Lisp HUG Maillist Archive

lispworks garbage collection thread

Unable to parse email body. Email id is 15414

Re: lispworks garbage collection thread

I’m curious about any answers too. But from the man pages:

It is normally less important to tune the ephemeral segments, that is the segments below the blocking generation. Functions that may be useful include set-default-segment-size, set-spare-keeping-policy and set-delay-promotion.

They also discuss the various profiling tools that can (should?) be used to track down where the problem is. I see from your code that you focus on Gen 2. Why? That is supposedly one of the ephemeral generations.

- DM

On Nov 9, 2020, at 6:15 PM, roy anderson <reanz1959@gmail.com> wrote:

We are developing a document analysis application which puts pressure on the LispWorks 64bit Windows Professional garbage collector. 
 
In particular, everything stops for quite a while the gc is working from time to time.
 
To reduce the impact of the gc. I am working on the code below to do automated garbage collection after either a certain number of tranactions or a set period.
 
This seems to help our situation but are we missing something?
 
Here’s our code:
 
(defun memory-magnitude (bytes)
  (let* ((kb (truncate bytes 1024))
         (mb (truncate kb 1024))
         (gb (truncate mb 1024))
         (tb (truncate gb 1024))
         (pb (truncate tb 1024)))
    (format nil "~{~a~}"
           (cond
            ((zerop kb) (list bytes "B"))
            ((zerop mb) (list kb "KB"))
            ((zerop gb) (list mb "MB"))
            ((zerop tb) (list gb "GB"))
            ((zerop pb) (list tb "TB"))
            (t (list pb "PB"))))))
 
(defun time-interval (start-transaction)
  (let* ((ms (truncate (* 1000 (/ (- (get-internal-real-time) start-transaction)
                                  internal-time-units-per-second))))
          (s (truncate ms 1000.0))
          (min (truncate s 60))
          (hours (truncate min 60))
          (days (truncate hours 24)))
    (format nil "~{~a~}"
            (cond
             ((zerop s) (list ms "msec"))
             ((zerop min) (list s "sec"))
             ((zerop hours) (list min "min"))
             ((zerop days) (list hours "hours"))
             (t (list days "days"))))))
 
(defun report-garbage-collection (thread-name actions last-collection)
  (let ((room (system:room-values)))
    (format t "~a[" thread-name)
    (unless (zerop actions)
      (format t "~d Action~:p, " actions))
    (format t "Slept: ~a] Total: ~a Free: ~a~%"
            (time-interval last-collection)
            (memory-magnitude (getf room :TOTAL-SIZE))
            (memory-magnitude (getf room :TOTAL-FREE)))))
 
(defparameter maximum-sleep-seconds (* 60 5)) ; i.e. 5 minutes
(defparameter transactions 0)
 
(defun make-garbage-manager ()
  (let ((top-level *standard-output*)
        (thread-name "garbage collector")
        (last-activity transactions)
        (last-collection (get-internal-real-time))
        (slept 0))
    (flet ((garbage-man ()
      (let ((*standard-output* top-level))
        (loop
            while t do
            (sleep 10)
            (incf slept 10)
            (let ((actions (- transactions last-activity)))
              (when (or (> actions 5)
                        (> slept maximum-sleep-seconds)) ; i.e. more than limit before gc
                (hcl:gc-generation 2)
                (report-garbage-collection thread-name actions last-collection)
                (setf last-collection (get-internal-real-time))
                (setf last-activity transactions)
                (setf slept 0)))))))
      (mp:process-run-function thread-name () #'garbage-man)))) 
 
(make-garbage-manager)
 
Sent from Mail for Windows 10
 
.+)�-�隊X�X�+.+)Z�����b�n�X� +��(�m��� b�(�K�o캚h����a�m

Re: lispworks garbage collection thread

Hi,

Maybe one of the potential ways to tackle the problem is to not to generate garbage? I.e. pre-allocate as many objects as needed on startup of the application, use object pools etc.

BR,
/Alexey

On Tue, Nov 10, 2020 at 2:17 AM roy anderson <reanz1959@gmail.com> wrote:

We are developing a document analysis application which puts pressure on the LispWorks 64bit Windows Professional garbage collector.

 

In particular, everything stops for quite a while the gc is working from time to time.

 

To reduce the impact of the gc. I am working on the code below to do automated garbage collection after either a certain number of tranactions or a set period.

 

This seems to help our situation but are we missing something?

 

Here’s our code:

 

(defun memory-magnitude (bytes)

  (let* ((kb (truncate bytes 1024))

         (mb (truncate kb 1024))

         (gb (truncate mb 1024))

         (tb (truncate gb 1024))

         (pb (truncate tb 1024)))

    (format nil "~{~a~}"

           (cond

            ((zerop kb) (list bytes "B"))

            ((zerop mb) (list kb "KB"))

            ((zerop gb) (list mb "MB"))

            ((zerop tb) (list gb "GB"))

            ((zerop pb) (list tb "TB"))

            (t (list pb "PB"))))))

 

(defun time-interval (start-transaction)

  (let* ((ms (truncate (* 1000 (/ (- (get-internal-real-time) start-transaction)

                                  internal-time-units-per-second))))

          (s (truncate ms 1000.0))

          (min (truncate s 60))

          (hours (truncate min 60))

          (days (truncate hours 24)))

    (format nil "~{~a~}"

            (cond

             ((zerop s) (list ms "msec"))

             ((zerop min) (list s "sec"))

             ((zerop hours) (list min "min"))

             ((zerop days) (list hours "hours"))

             (t (list days "days"))))))

 

(defun report-garbage-collection (thread-name actions last-collection)

  (let ((room (system:room-values)))

    (format t "~a[" thread-name)

    (unless (zerop actions)

      (format t "~d Action~:p, " actions))

    (format t "Slept: ~a] Total: ~a Free: ~a~%"

            (time-interval last-collection)

            (memory-magnitude (getf room :TOTAL-SIZE))

            (memory-magnitude (getf room :TOTAL-FREE)))))

 

(defparameter maximum-sleep-seconds (* 60 5)) ; i.e. 5 minutes

(defparameter transactions 0)

 

(defun make-garbage-manager ()

  (let ((top-level *standard-output*)

        (thread-name "garbage collector")

        (last-activity transactions)

        (last-collection (get-internal-real-time))

        (slept 0))

    (flet ((garbage-man ()

      (let ((*standard-output* top-level))

        (loop

            while t do

            (sleep 10)

            (incf slept 10)

            (let ((actions (- transactions last-activity)))

              (when (or (> actions 5)

                        (> slept maximum-sleep-seconds)) ; i.e. more than limit before gc

                (hcl:gc-generation 2)

                (report-garbage-collection thread-name actions last-collection)

                (setf last-collection (get-internal-real-time))

                (setf last-activity transactions)

                (setf slept 0)))))))

      (mp:process-run-function thread-name () #'garbage-man))))

 

(make-garbage-manager)

 

Sent from Mail for Windows 10

 

Re: lispworks garbage collection thread

Hi!

At Netfonds, we had lots of heavy server applications that would run for months non stop, and they all had our own gc management code built in. 

This consisted of:
1) “Cron in lisp” - a lisp thread similar to the cron Unix tool which was used to run programs at regular intervals
2) A gc function called from cron which checked the amount of memory used, and decided (according to limits set by the app) whether it’s time for a global GC - which btw is generation 3, not 2 (we also on some occasions collected gen 7, but that’s not useful on a regular schedule). 

For most of the apps, the gc function would typically run once or twice per day. Since we were a stock trading provider, we would typically have vast amounts of market data from yesterday that turned into garbage each morning, so in some apps a morning gc was run regardless of the memory usage monitoring. 

Hope this helps!

Regards,
  Espen Vestre

10. nov. 2020 kl. 02:17 skrev roy anderson <reanz1959@gmail.com>:



We are developing a document analysis application which puts pressure on the LispWorks 64bit Windows Professional garbage collector.

 

In particular, everything stops for quite a while the gc is working from time to time.

 

To reduce the impact of the gc. I am working on the code below to do automated garbage collection after either a certain number of tranactions or a set period.

 

This seems to help our situation but are we missing something?

 

Here’s our code:

 

(defun memory-magnitude (bytes)

В  (let* ((kb (truncate bytes 1024))

В В В В В В В В  (mb (truncate kb 1024))

В В В В В В В В  (gb (truncate mb 1024))

В В В В В В В В  (tb (truncate gb 1024))

В В В В В В В В  (pb (truncate tb 1024)))

В В В  (format nil "~{~a~}"

В В В В В В В В В В  (cond

В В В В В В В В В В В  ((zerop kb) (list bytes "B"))

В В В В В В В В В В В  ((zerop mb) (list kb "KB"))

В В В В В В В В В В В  ((zerop gb) (list mb "MB"))

В В В В В В В В В В В  ((zerop tb) (list gb "GB"))

В В В В В В В В В В В  ((zerop pb) (list tb "TB"))

В В В В В В В В В В В  (t (list pb "PB"))))))

 

(defun time-interval (start-transaction)

В  (let* ((ms (truncate (* 1000 (/ (- (get-internal-real-time) start-transaction)

В В В В В В В В В В В В В В В В В В В В В В В В В В В В В В В В В  internal-time-units-per-second))))

В В В В В В В В В  (s (truncate ms 1000.0))

В В В В В В В В В  (min (truncate s 60))

В В В В В В В В В  (hours (truncate min 60))

В В В В В В В В В  (days (truncate hours 24)))

В В В  (format nil "~{~a~}"

В В В В В В В В В В В  (cond

В В В В В В В В В В В В  ((zerop s) (list ms "msec"))

В В В  В В В В В В В В В ((zerop min) (list s "sec"))

В В В В В В В В В В В В  ((zerop hours) (list min "min"))

В В В В В В В В В В В В  ((zerop days) (list hours "hours"))

В В В В В В В В В В В В  (t (list days "days"))))))

 

(defun report-garbage-collection (thread-name actions last-collection)

В  (let ((room (system:room-values)))

В В В  (format t "~a[" thread-name)

В В В  (unless (zerop actions)

В В В В В  (format t "~d Action~:p, " actions))

В В В  (format t "Slept: ~a] Total: ~a Free: ~a~%"

В В В В В В В В В В В  (time-interval last-collection)

В В В В В В В В В В В  (memory-magnitude (getf room :TOTAL-SIZE))

В В В В В В В В В В В  (memory-magnitude (getf room :TOTAL-FREE)))))

 

(defparameter maximum-sleep-seconds (* 60 5)) ; i..e. 5 minutes

(defparameter transactions 0)

 

(defun make-garbage-manager ()

В  (let ((top-level *standard-output*)

В В В В В В В  (thread-name "garbage collector")

В В В В В В В  (last-activity transactions)

В В В В В В В  (last-collection (get-internal-real-time))

В В В В В В В  (slept 0))

В В В  (flet ((garbage-man ()

В В В В В  (let ((*standard-output* top-level))

В В В В В В В  (loop

В В В В В В В В В В В  while t do

В В В В В В В В В В В  (sleep 10)

В В В В В В В В В В В  (incf slept 10)

В В В В В В В В В В В  (let ((actions (- transactions last-activity)))

В В В В В В В В В В В В В  (when (or (> actions 5)

В В В В В В В В В В В В В В В В В В В В В В В  (> slept maximum-sleep-seconds)) ; i.e. more than limit before gc

В В В В В В В В В В В В В В В  (hcl:gc-generation 2)

В В В В В В В В В В В В В В В  (report-garbage-collection thread-name actions last-collection)

В В В В В В В В В В В В В В В  (setf last-collection (get-internal-real-time))

В В В В В В В В В В В В В В В  (setf last-activity transactions)

В В В В В В В В В В В В В В В  (setf slept 0)))))))

В В В В В  (mp:process-run-function thread-name () #'garbage-man))))

 

(make-garbage-manager)

 

Sent from Mail for Windows 10

 

вІ‘о‚Ш^™ЁҐЉx%ЉЛ_ўІвІ•Ё®K.±км–+)†и%ЉКpў№,r‰Ў¶Ъя 0–+)ВЉд±К&юЛ©¦ЉнюX¬¦ †ЩҐ

RE: lispworks garbage collection thread

Calling hcl:gc-generation  with generation 3 with the thread below causes frequent medium duration GC calls. Still better than the default periodic “stop the world” GC.

 

Now testing GC generation 2.

 

From: cjv
Sent: Wednesday, 11 November 2020 4:55 AM
To: Alexey Veretennikov
Cc: roy anderson; lisp-hug@lispworks.com; Peter Smyth
Subject: Re: lispworks garbage collection thread

 

I concur with Alexey.  My experience has been that futzing around with gc parameters is a waste of time.  Getting you application to stop consing is where you can really improve gc performance.  



On Nov 10, 2020, at 12:41 AM, Alexey Veretennikov <txm.fourier@gmail.com> wrote:

 

Hi,

 

Maybe one of the potential ways to tackle the problem is to not to generate garbage? I.e. pre-allocate as many objects as needed on startup of the application, use object pools etc.

 

BR,

/Alexey

 

On Tue, Nov 10, 2020 at 2:17 AM roy anderson <reanz1959@gmail.com> wrote:

We are developing a document analysis application which puts pressure on the LispWorks 64bit Windows Professional garbage collector.

 

In particular, everything stops for quite a while the gc is working from time to time.

 

To reduce the impact of the gc. I am working on the code below to do automated garbage collection after either a certain number of tranactions or a set period.

 

This seems to help our situation but are we missing something?

 

Here’s our code:

 

(defun memory-magnitude (bytes)

  (let* ((kb (truncate bytes 1024))

         (mb (truncate kb 1024))

         (gb (truncate mb 1024))

         (tb (truncate gb 1024))

         (pb (truncate tb 1024)))

    (format nil "~{~a~}"

           (cond

            ((zerop kb) (list bytes "B"))

            ((zerop mb) (list kb "KB"))

            ((zerop gb) (list mb "MB"))

            ((zerop tb) (list gb "GB"))

            ((zerop pb) (list tb "TB"))

            (t (list pb "PB"))))))

 

(defun time-interval (start-transaction)

  (let* ((ms (truncate (* 1000 (/ (- (get-internal-real-time) start-transaction)

                                  internal-time-units-per-second))))

          (s (truncate ms 1000.0))

          (min (truncate s 60))

          (hours (truncate min 60))

          (days (truncate hours 24)))

    (format nil "~{~a~}"

            (cond

             ((zerop s) (list ms "msec"))

             ((zerop min) (list s "sec"))

             ((zerop hours) (list min "min"))

             ((zerop days) (list hours "hours"))

             (t (list days "days"))))))

 

(defun report-garbage-collection (thread-name actions last-collection)

  (let ((room (system:room-values)))

    (format t "~a[" thread-name)

    (unless (zerop actions)

      (format t "~d Action~:p, " actions))

    (format t "Slept: ~a] Total: ~a Free: ~a~%"

            (time-interval last-collection)

            (memory-magnitude (getf room :TOTAL-SIZE))

            (memory-magnitude (getf room :TOTAL-FREE)))))

 

(defparameter maximum-sleep-seconds (* 60 5)) ; i.e. 5 minutes

(defparameter transactions 0)

 

(defun make-garbage-manager ()

  (let ((top-level *standard-output*)

        (thread-name "garbage collector")

        (last-activity transactions)

        (last-collection (get-internal-real-time))

        (slept 0))

    (flet ((garbage-man ()

      (let ((*standard-output* top-level))

        (loop

            while t do

            (sleep 10)

            (incf slept 10)

            (let ((actions (- transactions last-activity)))

              (when (or (> actions 5)

                        (> slept maximum-sleep-seconds)) ; i.e. more than limit before gc

                (hcl:gc-generation 2)

                (report-garbage-collection thread-name actions last-collection)

                (setf last-collection (get-internal-real-time))

                (setf last-activity transactions)

                (setf slept 0)))))))

      (mp:process-run-function thread-name () #'garbage-man))))

 

(make-garbage-manager)

 

Sent from Mail for Windows 10

 

 

 

_______________________________________________ Lisp Hug - the mailing list for LispWorks users lisp-hug@lispworks.com http://www.lispworks.com/support/lisp-hug.html

RE: lispworks garbage collection thread

My computer has 16GB RAM

 

On starting Lispworks:

 

CL-USER 3 > (system:room-values)

(:TOTAL-SIZE 121634816 :TOTAL-ALLOCATED 42347120 :TOTAL-FREE 76461456)

 

After the application has completed caching

 

CL-USER 7 > (system:room-values)

(:TOTAL-SIZE 4795203584 :TOTAL-ALLOCATED 4017765336 :TOTAL-FREE 637015080)

 

First output from the gc thread:

 

garbage collector[Slept: 5min] Total: 4GB Free: 662MB

 

Here is room full immediately after starting LispWorks

 

CL-USER 2 > (room :full)

> Generation 7: 34755920 (0x2125550)

       Cons               5047088 (0x4D0330)

       Non-Pointer        3045240 (0x2E7778)

       Other              8532072 (0x823068)

       Symbol             3130272 (0x2FC3A0)

       Function           14697672 (0xE044C8)

       Non-Pointer-Static 21120 (0x5280)

       Mixed-Static       281776 (0x44CB0)

       Weak               680 (0x2A8)

> Generation 6: 0 (0x0)

> Generation 5: 0 (0x0)

> Generation 4: 0 (0x0)

> Generation 3: 13792 (0x35E0)

       Non-Pointer        5824 (0x16C0)

       Symbol             7968 (0x1F20)

> Generation 2: 3058864 (0x2EACB0)

       Non-Pointer        2097160 (0x200008)

       Non-Pointer-Static 961704 (0xEACA8)

> Generation 1: 1592160 (0x184B60)

       Cons               431472 (0x69570)

       Non-Pointer        53536 (0xD120)

       Other              940576 (0xE5A20)

       Symbol             96 (0x60)

       Function           166320 (0x289B0)

       Weak               160 (0xA0)

> Generation 0: 1747632 (0x1AAAB0)

       Cons               858048 (0xD17C0)

       Non-Pointer        176096 (0x2AFE0)

       Other              681168 (0xA64D0)

       Symbol             288 (0x120)

       Function           31968 (0x7CE0)

       Weak               64 (0x40)

 

Total allocated 41168368 (0x2742DF0), total size 121634816 (0x7400000)

 

And here’s room after loading the applications cache

 

CL-USER 2 > (room :full)

> Generation 7: 34758720 (0x2126040)

       Cons               5047088 (0x4D0330)

       Non-Pointer        3045240 (0x2E7778)

       Other              8532072 (0x823068)

       Symbol             3130272 (0x2FC3A0)

       Function           14697672 (0xE044C8)

       Non-Pointer-Static 21120 (0x5280)

       Mixed-Static       284576 (0x457A0)

       Weak               680 (0x2A8)

> Generation 6: 0 (0x0)

> Generation 5: 0 (0x0)

> Generation 4: 0 (0x0)

> Generation 3: 2941556808 (0xAF549848)

       Cons               668503360 (0x27D88D40)

       Non-Pointer        1151455496 (0x44A1D108)

       Other              1113444848 (0x425DD1F0)

       Symbol             1044096 (0xFEE80)

       Function           7107616 (0x6C7420)

       Weak               1392 (0x570)

> Generation 2: 810159440 (0x304A0D50)

       Cons               185817632 (0xB135A20)

       Non-Pointer        365137104 (0x15C38CD0)

       Other              257178344 (0xF543AE8)

       Function           32 (0x20)

       Non-Pointer-Static 2026328 (0x1EEB58)

> Generation 1: 319100400 (0x130515F0)

       Cons               97253280 (0x5CBF7A0)

       Non-Pointer        70191936 (0x42F0B40)

       Other              151654768 (0x90A1170)

       Function           416 (0x1A0)

> Generation 0: 32799616 (0x1F47B80)

       Cons               11761568 (0xB377A0)

       Non-Pointer        1156088 (0x11A3F8)

       Other              19807880 (0x12E3E88)

       Function           74080 (0x12160)

 

Total allocated 4138374984 (0xF6AA9748), total size 5316214784 (0x13CDF0000)

 

 

From: cjv
Sent: Friday, 13 November 2020 10:31 AM
To: roy anderson
Cc: Alexey Veretennikov; lisp-hug@lispworks.com; Peter Smyth
Subject: Re: lispworks garbage collection thread

 

Are you paging?  

What is the size of your physical memory, and what is the size of your process?  

It bothers me that you say “stop the world GC” as that isn’t supposed to happen.

Have you done any profiling, like using extended-time for example?

 



On Nov 12, 2020, at 12:29 PM, roy anderson <reanz1959@gmail.com> wrote:

 

Calling hcl:gc-generation  with generation 3 with the thread below causes frequent medium duration GC calls. Still better than the default periodic “stop the world” GC.

 

Now testing GC generation 2.

 

From: cjv
Sent: Wednesday, 11 November 2020 4:55 AM
To: Alexey Veretennikov
Cc: roy anderson; lisp-hug@lispworks.com; Peter Smyth
Subject: Re: lispworks garbage collection thread

 

I concur with Alexey.  My experience has been that futzing around with gc parameters is a waste of time.  Getting you application to stop consing is where you can really improve gc performance.  




On Nov 10, 2020, at 12:41 AM, Alexey Veretennikov <txm.fourier@gmail.com> wrote:

 

Hi,

 

Maybe one of the potential ways to tackle the problem is to not to generate garbage? I.e. pre-allocate as many objects as needed on startup of the application, use object pools etc. 

 

BR,

/Alexey

 

On Tue, Nov 10, 2020 at 2:17 AM roy anderson <reanz1959@gmail.com> wrote:

We are developing a document analysis application which puts pressure on the LispWorks 64bit Windows Professional garbage collector.. 

 

In particular, everything stops for quite a while the gc is working from time to time.

 

To reduce the impact of the gc. I am working on the code below to do automated garbage collection after either a certain number of tranactions or a set period..

 

This seems to help our situation but are we missing something?

 

Here’s our code:

 

(defun memory-magnitude (bytes)

  (let* ((kb (truncate bytes 1024))

         (mb (truncate kb 1024))

         (gb (truncate mb 1024))

         (tb (truncate gb 1024))

         (pb (truncate tb 1024)))

    (format nil "~{~a~}"

           (cond

            ((zerop kb) (list bytes "B"))

            ((zerop mb) (list kb "KB"))

            ((zerop gb) (list mb "MB"))

            ((zerop tb) (list gb "GB"))

            ((zerop pb) (list tb "TB"))

            (t (list pb "PB"))))))

 

(defun time-interval (start-transaction)

  (let* ((ms (truncate (* 1000 (/ (- (get-internal-real-time) start-transaction)

                                  internal-time-units-per-second))))

          (s (truncate ms 1000.0))

          (min (truncate s 60))

          (hours (truncate min 60))

          (days (truncate hours 24)))

    (format nil "~{~a~}"

            (cond

             ((zerop s) (list ms "msec"))

             ((zerop min) (list s "sec"))

             ((zerop hours) (list min "min"))

             ((zerop days) (list hours "hours"))

             (t (list days "days"))))))

 

(defun report-garbage-collection (thread-name actions last-collection)

  (let ((room (system:room-values)))

    (format t "~a[" thread-name)

    (unless (zerop actions)

      (format t "~d Action~:p, " actions))

    (format t "Slept: ~a] Total: ~a Free: ~a~%"

            (time-interval last-collection)

            (memory-magnitude (getf room :TOTAL-SIZE))

            (memory-magnitude (getf room :TOTAL-FREE)))))

 

(defparameter maximum-sleep-seconds (* 60 5)) ; i.e. 5 minutes

(defparameter transactions 0)

 

(defun make-garbage-manager ()

  (let ((top-level *standard-output*)

        (thread-name "garbage collector")

        (last-activity transactions)

        (last-collection (get-internal-real-time))

        (slept 0))

    (flet ((garbage-man ()

      (let ((*standard-output* top-level))

        (loop

            while t do

            (sleep 10)

            (incf slept 10)

            (let ((actions (- transactions last-activity)))

              (when (or (> actions 5)

                        (> slept maximum-sleep-seconds)) ; i.e. more than limit before gc

                (hcl:gc-generation 2)

                (report-garbage-collection thread-name actions last-collection)

                (setf last-collection (get-internal-real-time))

                (setf last-activity transactions)

                (setf slept 0)))))))

      (mp:process-run-function thread-name () #'garbage-man)))) 

 

(make-garbage-manager)

 

Sent from Mail for Windows 10

 

 

 

_______________________________________________ Lisp Hug - the mailing list for LispWorks users lisp-hug@lispworks.com http://www.lispworks.com/support/lisp-hug.html

 

 

_______________________________________________ Lisp Hug - the mailing list for LispWorks users lisp-hug@lispworks.com http://www.lispworks.com/support/lisp-hug.html

Re: lispworks garbage collection thread

You should use the output of ROOM-VALUES to decide whether a GC of generation 3 really is necessary. There’s a cost to calling ROOM-VALUES, too, so it shouldn’t be called too frequently, but there’s no need to run a GC of the blocking generation when there’s not much to collect.

Espen

12. nov. 2020 kl. 21:29 skrev roy anderson <reanz1959@gmail.com>:

Calling hcl:gc-generation  with generation 3 with the thread below causes frequent medium duration GC calls. Still better than the default periodic “stop the world” GC.
 
Now testing GC generation 2.
 
From: cjv
Sent: Wednesday, 11 November 2020 4:55 AM
To: Alexey Veretennikov
Cc: roy anderson; lisp-hug@lispworks.com; Peter Smyth
Subject: Re: lispworks garbage collection thread
 
I concur with Alexey.  My experience has been that futzing around with gc parameters is a waste of time.  Getting you application to stop consing is where you can really improve gc performance.  


On Nov 10, 2020, at 12:41 AM, Alexey Veretennikov <txm.fourier@gmail.com> wrote:
 
Hi,
 
Maybe one of the potential ways to tackle the problem is to not to generate garbage? I.e. pre-allocate as many objects as needed on startup of the application, use object pools etc. 
 
BR,
/Alexey
 
On Tue, Nov 10, 2020 at 2:17 AM roy anderson <reanz1959@gmail.com> wrote:
We are developing a document analysis application which puts pressure on the LispWorks 64bit Windows Professional garbage collector. 

 

In particular, everything stops for quite a while the gc is working from time to time.

 

To reduce the impact of the gc. I am working on the code below to do automated garbage collection after either a certain number of tranactions or a set period.

 

This seems to help our situation but are we missing something?

 

Here’s our code:

 

(defun memory-magnitude (bytes)
  (let* ((kb (truncate bytes 1024))
         (mb (truncate kb 1024))
         (gb (truncate mb 1024))
         (tb (truncate gb 1024))
         (pb (truncate tb 1024)))
    (format nil "~{~a~}"
           (cond
            ((zerop kb) (list bytes "B"))
            ((zerop mb) (list kb "KB"))
            ((zerop gb) (list mb "MB"))
            ((zerop tb) (list gb "GB"))
            ((zerop pb) (list tb "TB"))
            (t (list pb "PB"))))))

 

(defun time-interval (start-transaction)
  (let* ((ms (truncate (* 1000 (/ (- (get-internal-real-time) start-transaction)
                                  internal-time-units-per-second))))
          (s (truncate ms 1000.0))
          (min (truncate s 60))
          (hours (truncate min 60))
          (days (truncate hours 24)))
    (format nil "~{~a~}"
            (cond
             ((zerop s) (list ms "msec"))
             ((zerop min) (list s "sec"))
             ((zerop hours) (list min "min"))
             ((zerop days) (list hours "hours"))
             (t (list days "days"))))))

 

(defun report-garbage-collection (thread-name actions last-collection)
  (let ((room (system:room-values)))
    (format t "~a[" thread-name)
    (unless (zerop actions)
      (format t "~d Action~:p, " actions))
    (format t "Slept: ~a] Total: ~a Free: ~a~%"
            (time-interval last-collection)
            (memory-magnitude (getf room :TOTAL-SIZE))
            (memory-magnitude (getf room :TOTAL-FREE)))))

 

(defparameter maximum-sleep-seconds (* 60 5)) ; i.e. 5 minutes
(defparameter transactions 0)

 

(defun make-garbage-manager ()
  (let ((top-level *standard-output*)
        (thread-name "garbage collector")
        (last-activity transactions)
        (last-collection (get-internal-real-time))
        (slept 0))
    (flet ((garbage-man ()
      (let ((*standard-output* top-level))
        (loop
            while t do
            (sleep 10)
            (incf slept 10)
            (let ((actions (- transactions last-activity)))
              (when (or (> actions 5)
                        (> slept maximum-sleep-seconds)) ; i.e. more than limit before gc
                (hcl:gc-generation 2)
                (report-garbage-collection thread-name actions last-collection)
                (setf last-collection (get-internal-real-time))
                (setf last-activity transactions)
                (setf slept 0)))))))
      (mp:process-run-function thread-name () #'garbage-man)))) 

 

(make-garbage-manager)

 

Sent from Mail for Windows 10

 

 
 
_______________________________________________ Lisp Hug - the mailing list for LispWorks users lisp-hug@lispworks.comhttp://www.lispworks.com/support/lisp-hug.html

Updated at: 2020-12-10 08:28 UTC