Memory usage optimization

pkg/rrc/helper.go

@@ -1,5 +1,12 @@
 package rrc
 
+import (
+	"runtime"
+
+	"github.com/dustin/go-humanize"
+	log "github.com/sirupsen/logrus"
+)
+
 func searchVL(V []float64, Sik float64) int {
 	left, right := 0, len(V)-2
 
@@ -24,3 +31,16 @@ func searchVL(V []float64, Sik float64) int {
 	// Return -1 if no such position is found
 	return -1
 }
+
+var lastTotalFreed uint64
+
+func printMemStats() {
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	log.Printf("[Memory] Alloc = %v TotalAlloc=%v Just Freed = %v Sys = %v NumGc=%v",
+		humanize.Bytes(m.Alloc),
+		humanize.Bytes(m.TotalAlloc),
+		humanize.Bytes(((m.TotalAlloc - m.Alloc) - lastTotalFreed)),
+		humanize.Bytes(m.Sys), m.NumGC)
+	lastTotalFreed = m.TotalAlloc - m.Alloc
+}

pkg/rrc/histogram.go

@@ -3,7 +3,6 @@ package rrc
 import (
 	"bytes"
 	"encoding/binary"
-	"fmt"
 	"math"
 	"os"
 
@@ -161,10 +160,16 @@ func (hist *ProbeHistogram) compute() {
 
 	log.Info("total Tij table entries:", nT)
 	if nT != int64(hist.probes*MaxGrayLevel) {
-		log.Warn("error in computing Tij table, some values are not satisfied")
+		log.Warn("error in computing Tij table, some values are not satisfied, nT:", nT,
+			"probes*MaxGrayLevel:", hist.probes*MaxGrayLevel)
 	}
 }
 
+const (
+	CheckPointProbe = 1000
+	CheckPointGray  = 15000
+)
+
 func (hist *ProbeHistogram) saveLUT(fLUT string) error {
 	file, err := os.Create(fLUT)
 	if err != nil {
@@ -173,16 +178,14 @@ func (hist *ProbeHistogram) saveLUT(fLUT string) error {
 	defer file.Close()
 	for i := 0; i < hist.probes; i++ {
 		binary.Write(file, binary.LittleEndian, hist.Tmat[i])
+		if i == CheckPointProbe {
+			log.Infof("Probes *, LUT check point [%d][%d]: %d", i, CheckPointGray, hist.Tmat[i][CheckPointGray])
+		}
 	}
 
 	return nil
 }
 
-const (
-	CheckPointProbe = 1000
-	CheckPointGray  = 15000
-)
-
 func LoadLUT(fLUT string, probes int) ([][]uint16, error) {
 	data, err := os.ReadFile(fLUT)
 	if err != nil {
@@ -214,7 +217,15 @@ func (hist *ProbeHistogram) sum(hists []*ProbeHistogram) {
 			hist.m_l[gray] += h.m_l[gray]
 		}
 	}
-
-	fmt.Println("Hist.M:", hist.M)
-	fmt.Println("Hist.probes:", hist.probes)
+}
+
+func (hist *ProbeHistogram) free() {
+	hist.N_i = nil
+	hist.n_ik = nil
+	hist.p_ik = nil
+	hist.m_l = nil
+	hist.P_l = nil
+	hist.S_ik = nil
+	hist.V_l = nil
+	hist.Tmat = nil
 }

pkg/rrc/histogram_test.go

@@ -0,0 +1,14 @@
+package rrc
+
+import "testing"
+
+func TestHistogram(t *testing.T) {
+	lut, err := LoadLUT("../../data/rrc/B0.LUT", PANCameraProbeNum)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if lut[CheckPointProbe][CheckPointGray] != 32767 {
+		t.Fail()
+	}
+}

pkg/rrc/rrc.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"runtime"
 	"sync"
 
 	log "github.com/sirupsen/logrus"
@@ -46,6 +47,7 @@ func (rrc *RRC) Close() {}
 
 // 统计探元灰度的累积概率密度
 func (rrc *RRC) StatisticalPAN(dsfile string) error {
+	printMemStats()
 	f, err := os.Open(dsfile)
 	if err != nil {
 		return err
@@ -61,7 +63,7 @@ func (rrc *RRC) StatisticalPAN(dsfile string) error {
 	// 并发处理
 	var wg sync.WaitGroup
 	jobs := make(chan struct{}, 5)
-	var hists []*ProbeHistogram
+
 	var mutex sync.Mutex
 	for _, file := range files {
 		wg.Add(1)
@@ -86,20 +88,26 @@ func (rrc *RRC) StatisticalPAN(dsfile string) error {
 			}
 
 			var hist ProbeHistogram
-			hist.init(PANCameraProbeNum)
+			hist.init0(PANCameraProbeNum)
 			hist.statistical(img)
 			img.Close()
 
 			mutex.Lock()
-			hists = append(hists, &hist)
+
+			log.Println("sum PAN histogram...")
+			rrc.Histograms[0].sum([]*ProbeHistogram{&hist})
+			hist.free()
+			printMemStats()
+			runtime.GC()
+			printMemStats()
+
 			mutex.Unlock()
 
 		}(file)
 	}
 
 	wg.Wait()
-	log.Println("sum PAN histogram...")
-	rrc.Histograms[0].sum(hists)
+
 	log.Println("compute PAN histogram...")
 	rrc.Histograms[0].compute()
 	log.Println("save PAN gray table matrix.")
@@ -109,6 +117,7 @@ func (rrc *RRC) StatisticalPAN(dsfile string) error {
 }
 
 func (rrc *RRC) StatisticalMSS(dsfile string) error {
+	printMemStats()
 	f, err := os.Open(dsfile)
 	if err != nil {
 		return err
@@ -124,7 +133,7 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error {
 
 	var wg sync.WaitGroup
 	jobs := make(chan struct{}, 5)
-	var hists [4][]*ProbeHistogram
+
 	var mutex sync.Mutex
 	for _, file := range files {
 		wg.Add(1)
@@ -141,7 +150,7 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error {
 			}
 
 			width := MSSCameraProbeNum
-			height := len(data) / (width * 2)
+			height := len(data) / (width * 4 * 2)
 			mssData := make([][]byte, 4)
 			for h := 0; h < height; h++ {
 				row := data[h*width*4*2 : (h+1)*width*4*2]
@@ -159,12 +168,19 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error {
 				}
 
 				var hist ProbeHistogram
-				hist.init(PANCameraProbeNum)
+				hist.init0(MSSCameraProbeNum)
 				hist.statistical(mssImages[i])
 				mssImages[i].Close()
 
 				mutex.Lock()
-				hists[i] = append(hists[i], &hist)
+
+				log.Println("sum MSS histogram...")
+				rrc.Histograms[i+1].sum([]*ProbeHistogram{&hist})
+				hist.free()
+				printMemStats()
+				runtime.GC()
+				printMemStats()
+
 				mutex.Unlock()
 			}
 		}(file)
@@ -172,8 +188,6 @@ func (rrc *RRC) StatisticalMSS(dsfile string) error {
 
 	wg.Wait()
 	for i := 1; i < 5; i++ {
-		log.Println("sum MSS histogram...")
-		rrc.Histograms[i].sum(hists[i-1])
 		log.Println("compute MSS histogram...")
 		rrc.Histograms[i].compute()
 		log.Println("save MSS gray table matrix.")