Memory usage optimization

cmd/rrc.go

@@ -17,12 +17,13 @@ var rrcCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		rrc := rrc.NewRRC()
 		rrc.StatisticalPAN(panDS)
+		rrc.StatisticalMSS(mssDS)
 	},
 }
 
 func init() {
 	rrcCmd.Flags().StringVarP(&panDS, "pan-dataset", "p", "data/RAW/pan.txt", "path to pan dataset")
-	rrcCmd.Flags().StringVarP(&mssDS, "mss-dataset", "m", "data/RAW/mss.txt", "path to mss dataset")
+	rrcCmd.Flags().StringVarP(&mssDS, "mss-dataset", "m", "data/RAW/mss-none.txt", "path to mss dataset")
 
 	rootCmd.AddCommand(rrcCmd)
 }

go.mod

@@ -18,6 +18,7 @@ require (
 	git.sr.ht/~sbinet/gg v0.5.0 // indirect
 	github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect
 	github.com/campoy/embedmd v1.0.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-fonts/liberation v0.3.2 // indirect
 	github.com/go-latex/latex v0.0.0-20231108140139-5c1ce85aa4ea // indirect
 	github.com/go-pdf/fpdf v0.9.0 // indirect

go.sum

@@ -851,6 +851,7 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3
 github.com/duke-git/lancet/v2 v2.3.1 h1:cYZHQp57CZKP41EFkV/7TGbUrmhjaPMI5vi3Q+9KJNo=
 github.com/duke-git/lancet/v2 v2.3.1/go.mod h1:zGa2R4xswg6EG9I6WnyubDbFO/+A/RROxIbXcwryTsc=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=

pkg/rrc/helper.go

@@ -1,5 +1,12 @@
 package rrc
 
+import (
+	"runtime"
+
+	"github.com/dustin/go-humanize"
+	log "github.com/sirupsen/logrus"
+)
+
 func searchVL(V []float64, Sik float64) int {
 	left, right := 0, len(V)-2
 
@@ -24,3 +31,16 @@ func searchVL(V []float64, Sik float64) int {
 	// Return -1 if no such position is found
 	return -1
 }
+
+var lastTotalFreed uint64
+
+func printMemStats() {
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	log.Printf("[Memory] Alloc = %v TotalAlloc=%v Just Freed = %v Sys = %v NumGc=%v",
+		humanize.Bytes(m.Alloc),
+		humanize.Bytes(m.TotalAlloc),
+		humanize.Bytes(((m.TotalAlloc - m.Alloc) - lastTotalFreed)),
+		humanize.Bytes(m.Sys), m.NumGC)
+	lastTotalFreed = m.TotalAlloc - m.Alloc
+}

pkg/rrc/histogram.go

@@ -3,7 +3,6 @@ package rrc
 import (
 	"bytes"
 	"encoding/binary"
-	"fmt"
 	"math"
 	"os"
 
@@ -161,10 +160,16 @@ func (hist *ProbeHistogram) compute() {
 
 	log.Info("total Tij table entries:", nT)
 	if nT != int64(hist.probes*MaxGrayLevel) {
-		log.Warn("error in computing Tij table, some values are not satisfied")
+		log.Warn("error in computing Tij table, some values are not satisfied, nT:", nT,
+			"probes*MaxGrayLevel:", hist.probes*MaxGrayLevel)
 	}
 }
 
+const (
+	CheckPointProbe = 1000
+	CheckPointGray  = 15000
+)
+
 func (hist *ProbeHistogram) saveLUT(fLUT string) error {
 	file, err := os.Create(fLUT)
 	if err != nil {
@@ -173,16 +178,14 @@ func (hist *ProbeHistogram) saveLUT(fLUT string) error {
 	defer file.Close()
 	for i := 0; i < hist.probes; i++ {
 		binary.Write(file, binary.LittleEndian, hist.Tmat[i])
+		if i == CheckPointProbe {
+			log.Infof("Probes *, LUT check point [%d][%d]: %d", i, CheckPointGray, hist.Tmat[i][CheckPointGray])
+		}
 	}
 
 	return nil
 }
 
-const (
-	CheckPointProbe = 1000
-	CheckPointGray  = 15000
-)
-
 func LoadLUT(fLUT string, probes int) ([][]uint16, error) {
 	data, err := os.ReadFile(fLUT)
 	if err != nil {
@@ -214,7 +217,15 @@ func (hist *ProbeHistogram) sum(hists []*ProbeHistogram) {
 			hist.m_l[gray] += h.m_l[gray]
 		}
 	}
-
+}
-	fmt.Println("Hist.M:", hist.M)
+
-	fmt.Println("Hist.probes:", hist.probes)
+func (hist *ProbeHistogram) free() {
+	hist.N_i = nil
+	hist.n_ik = nil
+	hist.p_ik = nil
+	hist.m_l = nil
+	hist.P_l = nil
+	hist.S_ik = nil
+	hist.V_l = nil
+	hist.Tmat = nil
 }

pkg/rrc/histogram_test.go

@@ -0,0 +1,14 @@
+package rrc
+
+import "testing"
+
+func TestHistogram(t *testing.T) {
+	lut, err := LoadLUT("../../data/rrc/B0.LUT", PANCameraProbeNum)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if lut[CheckPointProbe][CheckPointGray] != 32767 {
+		t.Fail()
+	}
+}

pkg/rrc/rrc.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"runtime"
 	"sync"
 
 	log "github.com/sirupsen/logrus"
@@ -46,6 +47,7 @@ func (rrc *RRC) Close() {}
 
 // 统计探元灰度的累积概率密度
 func (rrc *RRC) StatisticalPAN(dsfile string) error {
+	printMemStats()
 	f, err := os.Open(dsfile)
 	if err != nil {
 		return err
@@ -61,7 +63,7 @@ func (rrc *RRC) StatisticalPAN(dsfile string) error {
 	// 并发处理
 	var wg sync.WaitGroup
 	jobs := make(chan struct{}, 5)
-	var hists []*ProbeHistogram
+
 	var mutex sync.Mutex
 	for _, file := range files {
 		wg.Add(1)
@@ -86,20 +88,26 @@ func (rrc *RRC) StatisticalPAN(dsfile string) error {
 			}
 
 			var hist ProbeHistogram
-			hist.init(PANCameraProbeNum)
+			hist.init0(PANCameraProbeNum)
 			hist.statistical(img)
 			img.Close()
 
 			mutex.Lock()
-			hists = append(hists, &hist)
+
+			log.Println("sum PAN histogram...")
+			rrc.Histograms[0].sum([]*ProbeHistogram{&hist})
+			hist.free()
+			printMemStats()
+			runtime.GC()
+			printMemStats()
+
 			mutex.Unlock()
 
 		}(file)
 	}
 
 	wg.Wait()
-	log.Println("sum PAN histogram...")
+
-	rrc.Histograms[0].sum(hists)
 	log.Println("compute PAN histogram...")
 	rrc.Histograms[0].compute()
 	log.Println("save PAN gray table matrix.")
@@ -109,6 +117,7 @@ func (rrc *RRC) StatisticalPAN(dsfile string) error {
 }
 
 func (rrc *RRC) StatisticalMSS(dsfile string) error {
+	printMemStats()
 	f, err := os.Open(dsfile)
 	if err != nil {
 		return err
@@ -124,7 +133,7 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error {
 
 	var wg sync.WaitGroup
 	jobs := make(chan struct{}, 5)
-	var hists [4][]*ProbeHistogram
+
 	var mutex sync.Mutex
 	for _, file := range files {
 		wg.Add(1)
@@ -141,7 +150,7 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error {
 			}
 
 			width := MSSCameraProbeNum
-			height := len(data) / (width * 2)
+			height := len(data) / (width * 4 * 2)
 			mssData := make([][]byte, 4)
 			for h := 0; h < height; h++ {
 				row := data[h*width*4*2 : (h+1)*width*4*2]
@@ -159,12 +168,19 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error {
 				}
 
 				var hist ProbeHistogram
-				hist.init(PANCameraProbeNum)
+				hist.init0(MSSCameraProbeNum)
 				hist.statistical(mssImages[i])
 				mssImages[i].Close()
 
 				mutex.Lock()
-				hists[i] = append(hists[i], &hist)
+
+				log.Println("sum MSS histogram...")
+				rrc.Histograms[i+1].sum([]*ProbeHistogram{&hist})
+				hist.free()
+				printMemStats()
+				runtime.GC()
+				printMemStats()
+
 				mutex.Unlock()
 			}
 		}(file)
@@ -172,8 +188,6 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error {
 
 	wg.Wait()
 	for i := 1; i < 5; i++ {
-		log.Println("sum MSS histogram...")
-		rrc.Histograms[i].sum(hists[i-1])
 		log.Println("compute MSS histogram...")
 		rrc.Histograms[i].compute()
 		log.Println("save MSS gray table matrix.")