only store valid pixels

extract/seperate.go

@@ -50,6 +50,7 @@ func (e *Extractor) SeprateAuxAndImgData(dataFile string, segmentIndex int) erro
 
 	var afh AuxFrameHead
 
+	pandata := make([]byte, 0)
 	msdata := make([][]byte, 4)
 	var header []byte
 	var ebAux []byte
@@ -107,19 +108,22 @@ func (e *Extractor) SeprateAuxAndImgData(dataFile string, segmentIndex int) erro
 		platAux = append(platAux, data[dataIndex:dataIndex+32]...)
 		dataIndex += 32
 
+		if !afh.B0 || !afh.B1 || !afh.B2 || !afh.B3 || !afh.B4 {
+			log.Error("not all bands are available")
+			break
+		}
+
 		// 存储图像数据到临时文件 - 以 ENVI BSQ 格式存储，同时提供 HDR 描述文件
 		if afh.B0 {
 			// wpan.Write(data[dataIndex : dataIndex+19040])
-			write16bPixelLittleEndian(lw.ws[PAN_RAW].w, data[dataIndex:dataIndex+19040])
+			pandata = append(pandata, data[dataIndex:dataIndex+19040]...)
 			dataIndex += 19040
-			panEnviHdr.Lines += 1
 		}
 
 		if afh.B1 {
 			// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
 			msdata[0] = append(msdata[0], data[dataIndex:dataIndex+1192]...)
 			dataIndex += 1192
-			mssEnviHdr.Lines += 1
 		}
 		if afh.B2 {
 			// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
@@ -139,53 +143,12 @@ func (e *Extractor) SeprateAuxAndImgData(dataFile string, segmentIndex int) erro
 		i = dataIndex // 完成一行数据解析
 	}
 
-	// var bands = 0
-	// for i := 0; i < 4; i++ {
-	// 	if len(msdata[i]) > 0 {
-	// 		log.Println("write mss data of band B", i+1)
-	// 		_, err := write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[i])
-	// 		if err != nil {
-	// 			log.Error("write mss data error:", err.Error())
-	// 		}
-	// 		bands += 1
-	// 	}
-	// }
-
 	if len(msdata[0]) != len(msdata[1]) || len(msdata[2]) != len(msdata[3]) {
 		log.Error("mss data of bands B1-B4 are not equal")
 		return errors.New("mss data of bands B1-B4 are not equal")
 	}
 
-	mssRowLen := 1192 * 4
+	// 先判断辅助数据是否完整
-	for i := 0; i < len(msdata[0]); i += mssRowLen {
-		var err error
-		_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[0][i:i+mssRowLen])
-		if err != nil {
-			log.Error("write mss 1 data error:", err.Error())
-		}
-		_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[1][i:i+mssRowLen])
-		if err != nil {
-			log.Error("write mss 2 data error:", err.Error())
-		}
-		_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[2][i:i+mssRowLen])
-		if err != nil {
-			log.Error("write mss 3 data error:", err.Error())
-		}
-		_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[3][i:i+mssRowLen])
-		if err != nil {
-			log.Error("write mss 4 data error:", err.Error())
-		}
-	}
-
-	panEnviHdr.Samples = 9520
-	panEnviHdr.Bands = 1
-	lw.ws[PAN_HDR].w.Write([]byte(panEnviHdr.String()))
-
-	mssEnviHdr.Lines = mssEnviHdr.Lines / 4 // 多光谱波段分别在 4 行中传输
-	mssEnviHdr.Samples = 2384 * 4
-	mssEnviHdr.Bands = 1
-	lw.ws[MSS_HDR].w.Write([]byte(mssEnviHdr.String()))
-
 	// 帧头+辅助数据
 	if len(header)/24 < len(ebAux)/128 || len(ebAux)/128 != len(platAux)/512 {
 		fmt.Println("aux data length:", len(header)/24, len(ebAux)/128, len(platAux)/512)
@@ -204,6 +167,64 @@ func (e *Extractor) SeprateAuxAndImgData(dataFile string, segmentIndex int) erro
 		return err
 	}
 
+	auxHeight := len(aps) * 16 // 16行图像数据为一组辅助数据
+	panHeight := len(pandata) / 19040
+	mssHeight := 4 * len(msdata[0]) / 4768
+	log.Println("pan height:", panHeight, "mss height:", mssHeight, "aux height:", auxHeight)
+	// 取最小值作为有效数据长度
+	efficientHeight := int(math.Min(float64(panHeight), float64(mssHeight)))
+	efficientHeight = int(math.Min(float64(efficientHeight), float64(auxHeight)))
+	log.Println("efficient height:", efficientHeight)
+
+	// 写入pan数据
+	for i := 0; i < len(pandata); i += 19040 {
+		start := i + (2+48+38)*2
+		end := start + 18688
+		write16bPixelLittleEndian(lw.ws[PAN_RAW].w, pandata[start:end])
+		panEnviHdr.Lines += 1
+		if panEnviHdr.Lines >= efficientHeight {
+			break
+		}
+	}
+
+	mssRowLen := 1192 * 4
+	for i := 0; i < len(msdata[0]); i += mssRowLen {
+		var err error
+		start := i + (1+11+10)*2
+		end := start + 4672
+		_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[0][start:end])
+		if err != nil {
+			log.Error("write mss 1 data error:", err.Error())
+		}
+		_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[1][start:end])
+		if err != nil {
+			log.Error("write mss 2 data error:", err.Error())
+		}
+		_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[2][start:end])
+		if err != nil {
+			log.Error("write mss 3 data error:", err.Error())
+		}
+		_, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[3][start:end])
+		if err != nil {
+			log.Error("write mss 4 data error:", err.Error())
+		}
+
+		mssEnviHdr.Lines += 1
+		if mssEnviHdr.Lines*4 >= efficientHeight {
+			break
+		}
+	}
+
+	panEnviHdr.Samples = 9344
+	panEnviHdr.Bands = 1
+	lw.ws[PAN_HDR].w.Write([]byte(panEnviHdr.String()))
+
+	mssEnviHdr.Samples = 2336 * 4
+	mssEnviHdr.Bands = 1
+	lw.ws[MSS_HDR].w.Write([]byte(mssEnviHdr.String()))
+
+	fmt.Println("pan height:", panEnviHdr.Lines, "mss height:", mssEnviHdr.Lines)
+
 	e.mutex.Lock()
 	defer e.mutex.Unlock()
 	seg := Segment{
@@ -310,3 +331,25 @@ func (e *Extractor) trimImgRawData(data []byte) ([]byte, *AuxFrameHead) {
 
 	return data[start:end], afh
 }
+
+// 9520 16bit 像素 19040 字节，返回有效像素字节起始位置
+// 9520 像素 2(Margin) + 48(OB) + 38(Margin) + 9344(Effective Pixels) + 38(Margin) + 48(OB) + 2(Margin)
+// 7056 像素 2(Margin) + 12(OB) + 18(Margin) + 20(Margin) + 7000(Effective Pixels) + 4(Margin)
+func PanEffectivePixel(row []byte) (start, end int) {
+	if len(row) < 19040 {
+		return -1, -1
+	}
+	start = (0 + 2 + 48 + 38) * 2
+	end = 19040 - (2+48+28)*2
+	return start, end
+}
+
+// 2384 像素 1+11+10+2336+10+8+8
+func MssEffectivePixel(row []byte) (start, end int) {
+	if len(row) < 4768 {
+		return -1, -1
+	}
+	start = (1 + 11 + 10) * 2
+	end = 4768 - (10+8+8)*2
+	return start, end
+}

extract/trans_frame.go

@@ -95,6 +95,9 @@ func (p *Extractor) ExtractOriginalImageData(aosDataFile string) ([]string, erro
 	}
 
 	log.Println("null transfer frame cnt:", nullFrmCnt)
+	if len(snRange) == 0 {
+		log.Errorf("no valid transfer frame found in %s", aosDataFile)
+	}
 
 	for k, v := range snRange {
 		vv := slice.Unique(v)

go.mod

@@ -2,7 +2,10 @@ module starwiz.cn/sjy01/preprocessing
 
 go 1.20
 
-require github.com/xuri/excelize/v2 v2.8.1
+require (
+	github.com/xuri/excelize/v2 v2.8.1
+	gocv.io/x/gocv v0.36.1
+)
 
 require (
 	github.com/jonboulle/clockwork v0.4.0 // indirect

go.sum

@@ -84,6 +84,8 @@ github.com/xuri/excelize/v2 v2.8.1/go.mod h1:oli1E4C3Pa5RXg1TBXn4ENCXDV5JUMlBluU
 github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 h1:qhbILQo1K3mphbwKh1vNm4oGezE1eF9fQWmNiIpSfI4=
 github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+gocv.io/x/gocv v0.36.1 h1:6XkEaPOk7h/umjy+MXgSEtSeCIgcPJhccUjrJFhjdTY=
+gocv.io/x/gocv v0.36.1/go.mod h1:lmS802zoQmnNvXETpmGriBqWrENPei2GxYx5KUxJsMA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

xlog/log.go

@@ -49,7 +49,7 @@ func init() {
 		DisableColors:   true,
 	}
 
-	configureLogger(logrus.StandardLogger(), "log/app.log", logrus.InfoLevel)
+	configureLogger(logrus.StandardLogger(), "log/SJY01-preproc.log", logrus.InfoLevel)
 }
 
 func NewLogger(logfile string) *logrus.Logger {