From 35bf364e97fdf847be3001140cac96e7007b1035 Mon Sep 17 00:00:00 2001 From: nuknal Date: Sat, 15 Jun 2024 13:05:42 +0800 Subject: [PATCH] Memory usage optimization --- cmd/rrc.go | 3 ++- go.mod | 1 + go.sum | 1 + pkg/rrc/helper.go | 20 ++++++++++++++++++++ pkg/rrc/histogram.go | 31 +++++++++++++++++++++---------- pkg/rrc/histogram_test.go | 14 ++++++++++++++ pkg/rrc/rrc.go | 36 +++++++++++++++++++++++++----------- 7 files changed, 84 insertions(+), 22 deletions(-) create mode 100644 pkg/rrc/histogram_test.go diff --git a/cmd/rrc.go b/cmd/rrc.go index d68575f..bf4135a 100644 --- a/cmd/rrc.go +++ b/cmd/rrc.go @@ -17,12 +17,13 @@ var rrcCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { rrc := rrc.NewRRC() rrc.StatisticalPAN(panDS) + rrc.StatisticalMSS(mssDS) }, } func init() { rrcCmd.Flags().StringVarP(&panDS, "pan-dataset", "p", "data/RAW/pan.txt", "path to pan dataset") - rrcCmd.Flags().StringVarP(&mssDS, "mss-dataset", "m", "data/RAW/mss.txt", "path to mss dataset") + rrcCmd.Flags().StringVarP(&mssDS, "mss-dataset", "m", "data/RAW/mss-none.txt", "path to mss dataset") rootCmd.AddCommand(rrcCmd) } diff --git a/go.mod b/go.mod index d16770e..a0e306f 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( git.sr.ht/~sbinet/gg v0.5.0 // indirect github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect github.com/campoy/embedmd v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-fonts/liberation v0.3.2 // indirect github.com/go-latex/latex v0.0.0-20231108140139-5c1ce85aa4ea // indirect github.com/go-pdf/fpdf v0.9.0 // indirect diff --git a/go.sum b/go.sum index 0c1ac68..cd7075f 100644 --- a/go.sum +++ b/go.sum @@ -851,6 +851,7 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/duke-git/lancet/v2 v2.3.1 h1:cYZHQp57CZKP41EFkV/7TGbUrmhjaPMI5vi3Q+9KJNo= github.com/duke-git/lancet/v2 v2.3.1/go.mod h1:zGa2R4xswg6EG9I6WnyubDbFO/+A/RROxIbXcwryTsc= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= diff --git a/pkg/rrc/helper.go b/pkg/rrc/helper.go index c6b27ed..4b9d3fa 100644 --- a/pkg/rrc/helper.go +++ b/pkg/rrc/helper.go @@ -1,5 +1,12 @@ package rrc +import ( + "runtime" + + "github.com/dustin/go-humanize" + log "github.com/sirupsen/logrus" +) + func searchVL(V []float64, Sik float64) int { left, right := 0, len(V)-2 @@ -24,3 +31,16 @@ func searchVL(V []float64, Sik float64) int { // Return -1 if no such position is found return -1 } + +var lastTotalFreed uint64 + +func printMemStats() { + var m runtime.MemStats + runtime.ReadMemStats(&m) + log.Printf("[Memory] Alloc = %v TotalAlloc=%v Just Freed = %v Sys = %v NumGc=%v", + humanize.Bytes(m.Alloc), + humanize.Bytes(m.TotalAlloc), + humanize.Bytes(((m.TotalAlloc - m.Alloc) - lastTotalFreed)), + humanize.Bytes(m.Sys), m.NumGC) + lastTotalFreed = m.TotalAlloc - m.Alloc +} diff --git a/pkg/rrc/histogram.go b/pkg/rrc/histogram.go index 0c4ca56..05c09bc 100644 --- a/pkg/rrc/histogram.go +++ b/pkg/rrc/histogram.go @@ -3,7 +3,6 @@ package rrc import ( "bytes" "encoding/binary" - "fmt" "math" "os" @@ -161,10 +160,16 @@ func (hist *ProbeHistogram) compute() { log.Info("total Tij table entries:", nT) if nT != int64(hist.probes*MaxGrayLevel) { - log.Warn("error in computing Tij table, some values are not satisfied") + log.Warn("error in computing Tij table, some values are not satisfied, nT:", nT, + "probes*MaxGrayLevel:", hist.probes*MaxGrayLevel) } } +const ( + CheckPointProbe = 1000 + CheckPointGray = 15000 +) + func (hist *ProbeHistogram) saveLUT(fLUT string) error { file, err := os.Create(fLUT) if err != nil { @@ -173,16 +178,14 @@ func (hist *ProbeHistogram) saveLUT(fLUT string) error { defer file.Close() for i := 0; i < hist.probes; i++ { binary.Write(file, binary.LittleEndian, hist.Tmat[i]) + if i == CheckPointProbe { + log.Infof("Probes *, LUT check point [%d][%d]: %d", i, CheckPointGray, hist.Tmat[i][CheckPointGray]) + } } return nil } -const ( - CheckPointProbe = 1000 - CheckPointGray = 15000 -) - func LoadLUT(fLUT string, probes int) ([][]uint16, error) { data, err := os.ReadFile(fLUT) if err != nil { @@ -214,7 +217,15 @@ func (hist *ProbeHistogram) sum(hists []*ProbeHistogram) { hist.m_l[gray] += h.m_l[gray] } } - - fmt.Println("Hist.M:", hist.M) - fmt.Println("Hist.probes:", hist.probes) +} + +func (hist *ProbeHistogram) free() { + hist.N_i = nil + hist.n_ik = nil + hist.p_ik = nil + hist.m_l = nil + hist.P_l = nil + hist.S_ik = nil + hist.V_l = nil + hist.Tmat = nil } diff --git a/pkg/rrc/histogram_test.go b/pkg/rrc/histogram_test.go new file mode 100644 index 0000000..6e34de7 --- /dev/null +++ b/pkg/rrc/histogram_test.go @@ -0,0 +1,14 @@ +package rrc + +import "testing" + +func TestHistogram(t *testing.T) { + lut, err := LoadLUT("../../data/rrc/B0.LUT", PANCameraProbeNum) + if err != nil { + t.Error(err) + } + + if lut[CheckPointProbe][CheckPointGray] != 32767 { + t.Fail() + } +} diff --git a/pkg/rrc/rrc.go b/pkg/rrc/rrc.go index 706d322..52d191f 100644 --- a/pkg/rrc/rrc.go +++ b/pkg/rrc/rrc.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "os" + "runtime" "sync" log "github.com/sirupsen/logrus" @@ -46,6 +47,7 @@ func (rrc *RRC) Close() {} // 统计探元灰度的累积概率密度 func (rrc *RRC) StatisticalPAN(dsfile string) error { + printMemStats() f, err := os.Open(dsfile) if err != nil { return err @@ -61,7 +63,7 @@ func (rrc *RRC) StatisticalPAN(dsfile string) error { // 并发处理 var wg sync.WaitGroup jobs := make(chan struct{}, 5) - var hists []*ProbeHistogram + var mutex sync.Mutex for _, file := range files { wg.Add(1) @@ -86,20 +88,26 @@ func (rrc *RRC) StatisticalPAN(dsfile string) error { } var hist ProbeHistogram - hist.init(PANCameraProbeNum) + hist.init0(PANCameraProbeNum) hist.statistical(img) img.Close() mutex.Lock() - hists = append(hists, &hist) + + log.Println("sum PAN histogram...") + rrc.Histograms[0].sum([]*ProbeHistogram{&hist}) + hist.free() + printMemStats() + runtime.GC() + printMemStats() + mutex.Unlock() }(file) } wg.Wait() - log.Println("sum PAN histogram...") - rrc.Histograms[0].sum(hists) + log.Println("compute PAN histogram...") rrc.Histograms[0].compute() log.Println("save PAN gray table matrix.") @@ -109,6 +117,7 @@ func (rrc *RRC) StatisticalPAN(dsfile string) error { } func (rrc *RRC) StatisticalMSS(dsfile string) error { + printMemStats() f, err := os.Open(dsfile) if err != nil { return err @@ -124,7 +133,7 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error { var wg sync.WaitGroup jobs := make(chan struct{}, 5) - var hists [4][]*ProbeHistogram + var mutex sync.Mutex for _, file := range files { wg.Add(1) @@ -141,7 +150,7 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error { } width := MSSCameraProbeNum - height := len(data) / (width * 2) + height := len(data) / (width * 4 * 2) mssData := make([][]byte, 4) for h := 0; h < height; h++ { row := data[h*width*4*2 : (h+1)*width*4*2] @@ -159,12 +168,19 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error { } var hist ProbeHistogram - hist.init(PANCameraProbeNum) + hist.init0(MSSCameraProbeNum) hist.statistical(mssImages[i]) mssImages[i].Close() mutex.Lock() - hists[i] = append(hists[i], &hist) + + log.Println("sum MSS histogram...") + rrc.Histograms[i+1].sum([]*ProbeHistogram{&hist}) + hist.free() + printMemStats() + runtime.GC() + printMemStats() + mutex.Unlock() } }(file) @@ -172,8 +188,6 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error { wg.Wait() for i := 1; i < 5; i++ { - log.Println("sum MSS histogram...") - rrc.Histograms[i].sum(hists[i-1]) log.Println("compute MSS histogram...") rrc.Histograms[i].compute() log.Println("save MSS gray table matrix.")