On Nov 9, 2020, at 6:15 PM, roy anderson <reanz1959@gmail.com> wrote:

We are developing a document analysis application which puts pressure on the LispWorks 64bit Windows Professional garbage collector. 

 

In particular, everything stops for quite a while the gc is working from time to time.

 

To reduce the impact of the gc. I am working on the code below to do automated garbage collection after either a certain number of tranactions or a set period.

 

This seems to help our situation but are we missing something?

 

Here’s our code:

 

(defun memory-magnitude (bytes)

  (let* ((kb (truncate bytes 1024))

         (mb (truncate kb 1024))

         (gb (truncate mb 1024))

         (tb (truncate gb 1024))

         (pb (truncate tb 1024)))

    (format nil "~{~a~}"

           (cond

            ((zerop kb) (list bytes "B"))

            ((zerop mb) (list kb "KB"))

            ((zerop gb) (list mb "MB"))

            ((zerop tb) (list gb "GB"))

            ((zerop pb) (list tb "TB"))

            (t (list pb "PB"))))))

 

(defun time-interval (start-transaction)

  (let* ((ms (truncate (* 1000 (/ (- (get-internal-real-time) start-transaction)

                                  internal-time-units-per-second))))

          (s (truncate ms 1000.0))

          (min (truncate s 60))

          (hours (truncate min 60))

          (days (truncate hours 24)))

    (format nil "~{~a~}"

            (cond

             ((zerop s) (list ms "msec"))

             ((zerop min) (list s "sec"))

             ((zerop hours) (list min "min"))

             ((zerop days) (list hours "hours"))

             (t (list days "days"))))))

 

(defun report-garbage-collection (thread-name actions last-collection)

  (let ((room (system:room-values)))

    (format t "~a[" thread-name)

    (unless (zerop actions)

      (format t "~d Action~:p, " actions))

    (format t "Slept: ~a] Total: ~a Free: ~a~%"

            (time-interval last-collection)

            (memory-magnitude (getf room :TOTAL-SIZE))

            (memory-magnitude (getf room :TOTAL-FREE)))))

 

(defparameter maximum-sleep-seconds (* 60 5)) ; i.e. 5 minutes

(defparameter transactions 0)

 

(defun make-garbage-manager ()

  (let ((top-level *standard-output*)

        (thread-name "garbage collector")

        (last-activity transactions)

        (last-collection (get-internal-real-time))

        (slept 0))

    (flet ((garbage-man ()

      (let ((*standard-output* top-level))

        (loop

            while t do

            (sleep 10)

            (incf slept 10)

            (let ((actions (- transactions last-activity)))

              (when (or (> actions 5)

                        (> slept maximum-sleep-seconds)) ; i.e. more than limit before gc

                (hcl:gc-generation 2)

                (report-garbage-collection thread-name actions last-collection)

                (setf last-collection (get-internal-real-time))

                (setf last-activity transactions)

                (setf slept 0)))))))

      (mp:process-run-function thread-name () #'garbage-man)))) 

 

(make-garbage-manager)

 

Sent from Mail for Windows 10

 

.+)�-�隊X�X�+.+)Z�����b�n�X� +��(�m��� b�(�K�o캚h����a�m

10. nov. 2020 kl. 02:17 skrev roy anderson <reanz1959@gmail.com>:

12. nov. 2020 kl. 21:29 skrev roy anderson <reanz1959@gmail.com>:

Calling hcl:gc-generation  with generation 3 with the thread below causes frequent medium duration GC calls. Still better than the default periodic “stop the world” GC.

 

Now testing GC generation 2.

 

From: cjv
Sent: Wednesday, 11 November 2020 4:55 AM
To: Alexey Veretennikov
Cc: roy anderson; lisp-hug@lispworks.com; Peter Smyth
Subject: Re: lispworks garbage collection thread

 

I concur with Alexey.  My experience has been that futzing around with gc parameters is a waste of time.  Getting you application to stop consing is where you can really improve gc performance.  

On Nov 10, 2020, at 12:41 AM, Alexey Veretennikov <txm.fourier@gmail.com> wrote:

 

Hi,

 

Maybe one of the potential ways to tackle the problem is to not to generate garbage? I.e. pre-allocate as many objects as needed on startup of the application, use object pools etc. 

 

BR,

/Alexey

 

On Tue, Nov 10, 2020 at 2:17 AM roy anderson <reanz1959@gmail.com> wrote:

We are developing a document analysis application which puts pressure on the LispWorks 64bit Windows Professional garbage collector. 

 

In particular, everything stops for quite a while the gc is working from time to time.

 

To reduce the impact of the gc. I am working on the code below to do automated garbage collection after either a certain number of tranactions or a set period.

 

This seems to help our situation but are we missing something?

 

Here’s our code:

 

(defun memory-magnitude (bytes)

  (let* ((kb (truncate bytes 1024))

         (mb (truncate kb 1024))

         (gb (truncate mb 1024))

         (tb (truncate gb 1024))

         (pb (truncate tb 1024)))

    (format nil "~{~a~}"

           (cond

            ((zerop kb) (list bytes "B"))

            ((zerop mb) (list kb "KB"))

            ((zerop gb) (list mb "MB"))

            ((zerop tb) (list gb "GB"))

            ((zerop pb) (list tb "TB"))

            (t (list pb "PB"))))))

 

(defun time-interval (start-transaction)

  (let* ((ms (truncate (* 1000 (/ (- (get-internal-real-time) start-transaction)

                                  internal-time-units-per-second))))

          (s (truncate ms 1000.0))

          (min (truncate s 60))

          (hours (truncate min 60))

          (days (truncate hours 24)))

    (format nil "~{~a~}"

            (cond

             ((zerop s) (list ms "msec"))

             ((zerop min) (list s "sec"))

             ((zerop hours) (list min "min"))

             ((zerop days) (list hours "hours"))

             (t (list days "days"))))))

 

(defun report-garbage-collection (thread-name actions last-collection)

  (let ((room (system:room-values)))

    (format t "~a[" thread-name)

    (unless (zerop actions)

      (format t "~d Action~:p, " actions))

    (format t "Slept: ~a] Total: ~a Free: ~a~%"

            (time-interval last-collection)

            (memory-magnitude (getf room :TOTAL-SIZE))

            (memory-magnitude (getf room :TOTAL-FREE)))))

 

(defparameter maximum-sleep-seconds (* 60 5)) ; i.e. 5 minutes

(defparameter transactions 0)

 

(defun make-garbage-manager ()

  (let ((top-level *standard-output*)

        (thread-name "garbage collector")

        (last-activity transactions)

        (last-collection (get-internal-real-time))

        (slept 0))

    (flet ((garbage-man ()

      (let ((*standard-output* top-level))

        (loop

            while t do

            (sleep 10)

            (incf slept 10)

            (let ((actions (- transactions last-activity)))

              (when (or (> actions 5)

                        (> slept maximum-sleep-seconds)) ; i.e. more than limit before gc

                (hcl:gc-generation 2)

                (report-garbage-collection thread-name actions last-collection)

                (setf last-collection (get-internal-real-time))

                (setf last-activity transactions)

                (setf slept 0)))))))

      (mp:process-run-function thread-name () #'garbage-man)))) 

 

(make-garbage-manager)

 

Sent from Mail for Windows 10

 

 

 

_______________________________________________ Lisp Hug - the mailing list for LispWorks users lisp-hug@lispworks.comhttp://www.lispworks.com/support/lisp-hug.html