refactor

extract/aux.go

@@ -1,14 +1,12 @@
 package extract
 
 import (
-	"bufio"
 	"encoding/binary"
 	"errors"
 	"fmt"
-	"io"
+	"math"
 	"os"
 	"path/filepath"
-	"sort"
 	"strings"
 	"time"
 
@@ -32,8 +30,8 @@ type AuxFrameHead struct {
 	// B0	全色状态	0：不输出1：输出
 	ImgMode     byte
 	SerialNo    uint32 //  当前流水号
-	TimeSec     uint32 //  时间（秒） 量纲：1us/bit，十六进制无符号整型数
+	TimeSec     uint32 //  时间（秒）
-	TimeSecFrac uint32 // 秒小数
+	TimeSecFrac uint32 // 秒小数 量纲：1us/bit，十六进制无符号整型数
 	FileNo      uint8  //  文件号
 
 	IsValidFrmHead bool
@@ -107,130 +105,36 @@ func (afh AuxFrameHead) CheckFrmHead() bool {
 	return false
 }
 
-type Record struct {
-	index   int
-	frmHead *AuxFrameHead
-}
-
 // 从传输帧文件中分离辅助数据，分别存储到辅助数据文件 _AUX.dat 和图像数据文件 _IMG_{波谱}.dat 中
-func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) error {
+func (e *Extractor) SeprateAuxAndImgData(dataFile string, segmentIndex int) error {
 	// 打开传输帧文件 - 一次读入内存
-	sData, err := os.ReadFile(sDataFile)
+	data, err := os.ReadFile(dataFile)
 	if err != nil {
-		log.Println("read data from", sDataFile, "error:", err.Error())
+		log.Println("read data from", dataFile, "error:", err.Error())
 		return err
 	}
 
-	log.Info("seprate aux and img data from", sDataFile)
+	log.Info("seprate aux and img data from", dataFile)
 
-	// 数据质量分析
+	name := filepath.Base(dataFile)
-	fimg, _ := os.Create("demo/temp/" + p.params.DataId + "_aux_img.txt")
-	defer fimg.Close()
-	fimg.WriteString("字节数 		帧头流水号 		文件号 		帧头时间 	中心辅助数据前4字节(行33-36)\n")
-	var qmap []*Record
-	var startAuxLine bool
-	var preSN uint32
-	for i := 0; i < len(sData); {
-		// 解析帧
-		if i+24 > len(sData) {
-			log.Info("length of original image frame head is not engough: ", len(sData)-i)
-			break
-		}
-
-		if sData[i] == 0xD1 && sData[i+1] == 0x5B && sData[i+2] == 0xD1 && sData[i+3] == 0x5B {
-			log.Debug("find package head: 0xD15BD15B")
-		} else {
-			i++
-			// log.Println(i,"not find package head: 0xD15BD15B, skip 1 byte")
-			continue
-		}
-
-		afh := &AuxFrameHead{}
-		afh.Decode(sData[i : i+24])
-
-		if !afh.IsValidFrmHead {
-			log.Debugf("[%d] invalid frame head of original raw data %v", i, afh.FrmHead)
-			i += 1
-			continue
-		}
-
-		r := &Record{index: i, frmHead: afh}
-		qmap = append(qmap, r)
-
-		utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36])
-
-		t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000)
-		tAux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0)
-
-		startAuxLine = tAux.Year() == t.Year() && tAux.Month() == t.Month() && tAux.Day() == t.Day() &&
-			tAux.Hour() == t.Hour() && tAux.Minute() == t.Minute()
-		if !startAuxLine {
-			// fimg.WriteString(fmt.Sprintf("-----%d not AUX Start -----\n", afh.SerialNo))
-		} else {
-			fimg.WriteString(fmt.Sprintf("%d 	%d 		%d 		%s 		%s\n",
-				i,
-				afh.SerialNo,
-				afh.FileNo,
-				t.String(),
-				tAux.String(),
-			))
-
-			if afh.SerialNo-preSN != 16 {
-				fmt.Println("serial number not continuous", afh.SerialNo, preSN)
-			}
-			preSN = afh.SerialNo
-		}
-
-		i += afh.RowLength
-	}
-
-	sort.Slice(qmap, func(i, j int) bool {
-		return qmap[i].index < qmap[j].index
-	})
-
-	name := filepath.Base(sDataFile)
 	name = strings.TrimRight(name, filepath.Ext(name))
-	outputDir := p.params.OutputPath
+	outputDir := e.params.OutputPath
 
-	aux0 := outputDir + "/" + name + "_EB.AUX"
+	lw := newL0Writer(outputDir, name)
-	os.Remove(aux0)
+	defer lw.Close()
-	faux0, _ := os.OpenFile(aux0, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
-	waux0 := bufio.NewWriter(faux0)
-	defer waux0.Flush()
-	defer faux0.Close()
 
-	aux1 := outputDir + "/" + name + "_PLAT.AUX"
+	var panEnviHdr, mssEnviHdr EnviHdr
-	os.Remove(aux1)
-	faux1, _ := os.OpenFile(aux1, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
-	waux1 := bufio.NewWriter(faux1)
-	defer waux1.Flush()
-	defer faux1.Close()
-
-	pan := outputDir + "/" + name + "_PAN.RAW"
-	os.Remove(pan)
-	fpan, _ := os.OpenFile(pan, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
-	wpan := bufio.NewWriter(fpan)
-	defer wpan.Flush()
-	defer fpan.Close()
-	panEnviHdr := EnviHdr{}
-	wpanHdr, _ := NewBSQWriter(outputDir+"/"+name+"_PAN.HDR", &panEnviHdr)
-	defer wpanHdr.Close()
-
-	// 先按单波段存储，再按波段组合存储为 BSQ 格式的 MSS
-	mss := outputDir + "/" + name + "_MSS.RAW"
-	os.Remove(mss)
-	fmss, _ := os.OpenFile(mss, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
-	wmss := bufio.NewWriter(fmss)
-	defer wmss.Flush()
-	defer fmss.Close()
-	mssEnviHdr := EnviHdr{}
-	wmssHdr, _ := NewBSQWriter(outputDir+"/"+name+"_MSS.HDR", &mssEnviHdr)
-	defer wmssHdr.Close()
 
 	var afh AuxFrameHead
-	dataLen := len(sData)
+	dataLen := len(data)
+	log.Info("original data length:", dataLen)
+	data = e.trimImgRawData(data)
+	dataLen = len(data)
+	log.Info("trimmed data length:", dataLen)
 
-	mssData := make([][]byte, 4)
+	e.quanlityAnalysis(data)
+
+	msdata := make([][]byte, 4)
 	var firstLineFound bool
 	var totalLines int
 	var pancount int
@@ -241,7 +145,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
 		}
 
 		// 包头
-		if sData[i] == 0xD1 && sData[i+1] == 0x5B && sData[i+2] == 0xD1 && sData[i+3] == 0x5B {
+		if data[i] == 0xD1 && data[i+1] == 0x5B && data[i+2] == 0xD1 && data[i+3] == 0x5B {
 			log.Debug("find package head: 0xD15BD15B")
 		} else {
 			i++
@@ -254,7 +158,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
 			log.Info("length of frame head is not engough for frame head")
 			break
 		}
-		afh.Decode(sData[i : i+24])
+		afh.Decode(data[i : i+24])
 		// pp.Println(afh)
 
 		if !afh.IsValidFrmHead {
@@ -269,7 +173,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
 			break
 		}
 
-		// 读取一行数据 sData[i : i+afh.RowLength]
+		// 读取一行数据 data[i : i+afh.RowLength]
 		if i+afh.RowLength > dataLen {
 			log.Infof("length of image row data %v is not enough: %v", dataLen-i, afh.RowLength)
 			break
@@ -277,7 +181,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
 
 		// 通过中心辅助时间判断是否第一行数据
 		if !firstLineFound {
-			utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36])
+			utcTime := binary.BigEndian.Uint32(data[i+32 : i+36])
 			t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000)
 			tAux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0)
 			firstLineFound = tAux.Year() == t.Year() && tAux.Month() == t.Month() && tAux.Day() == t.Day() &&
@@ -292,39 +196,39 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
 
 		// 存储辅助数据到临时文件
 		dataIndex := i + 24
-		waux0.Write(sData[dataIndex : dataIndex+8]) // 8字节焦面电箱辅助数据
+		lw.ws[EB_AUX].w.Write(data[dataIndex : dataIndex+8]) // 8字节焦面电箱辅助数据
 		dataIndex += 8
-		waux1.Write(sData[dataIndex : dataIndex+32]) // 32字节中心机辅助数据
+		lw.ws[PLAT_AUX].w.Write(data[dataIndex : dataIndex+32]) // 32字节中心机辅助数据
 		dataIndex += 32
 
 		// 存储图像数据到临时文件 - 以 ENVI BSQ 格式存储，同时提供 HDR 描述文件
 		if afh.B0 {
-			// wpan.Write(sData[dataIndex : dataIndex+19040])
+			// wpan.Write(data[dataIndex : dataIndex+19040])
-			write16bPixelLittleEndian(wpan, sData[dataIndex:dataIndex+19040])
+			write16bPixelLittleEndian(lw.ws[PAN_RAW].w, data[dataIndex:dataIndex+19040])
 			dataIndex += 19040
 			panEnviHdr.Lines += 1
 			pancount += 19040
 		}
 
 		if afh.B1 {
-			// write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192])
+			// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
-			mssData[0] = append(mssData[0], sData[dataIndex:dataIndex+1192]...)
+			msdata[0] = append(msdata[0], data[dataIndex:dataIndex+1192]...)
 			dataIndex += 1192
 			mssEnviHdr.Lines += 1
 		}
 		if afh.B2 {
-			// write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192])
+			// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
-			mssData[1] = append(mssData[1], sData[dataIndex:dataIndex+1192]...)
+			msdata[1] = append(msdata[1], data[dataIndex:dataIndex+1192]...)
 			dataIndex += 1192
 		}
 		if afh.B3 {
-			// write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192])
+			// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
-			mssData[2] = append(mssData[2], sData[dataIndex:dataIndex+1192]...)
+			msdata[2] = append(msdata[2], data[dataIndex:dataIndex+1192]...)
 			dataIndex += 1192
 		}
 		if afh.B4 {
-			// write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192])
+			// write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192])
-			mssData[3] = append(mssData[3], sData[dataIndex:dataIndex+1192]...)
+			msdata[3] = append(msdata[3], data[dataIndex:dataIndex+1192]...)
 			dataIndex += 1192
 		}
 		i = dataIndex // 完成一行数据解析
@@ -332,9 +236,9 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
 
 	var bands = 0
 	for i := 0; i < 4; i++ {
-		if len(mssData[i]) > 0 {
+		if len(msdata[i]) > 0 {
 			log.Println("write mss data of band B", i+1)
-			// _, err := write16bPixelLittleEndian(wmss, mssData[i])
+			_, err := write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[i])
 			if err != nil {
 				log.Error("write mss data error:", err.Error())
 			}
@@ -342,24 +246,104 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err
 		}
 	}
 
-	write16bPixelLittleEndian(wmss, mssData[2])
-
 	panEnviHdr.Samples = 9520
 	panEnviHdr.Bands = 1
-	wpanHdr.Write([]byte(panEnviHdr.String()))
+	lw.ws[PAN_HDR].w.Write([]byte(panEnviHdr.String()))
 
 	mssEnviHdr.Lines = mssEnviHdr.Lines / 4 // 多光谱波段分别在 4 行中传输
 	mssEnviHdr.Samples = 2384
 	mssEnviHdr.Bands = 1
-	wmssHdr.Write([]byte(mssEnviHdr.String()))
+	lw.ws[MSS_HDR].w.Write([]byte(mssEnviHdr.String()))
 
 	return nil
 }
 
-// 写入 16 位像素值，小端序
+func (e *Extractor) quanlityAnalysis(data []byte) {
-func write16bPixelLittleEndian(w io.Writer, data []byte) (int, error) {
+	// 数据质量分析
-	for i := 0; i < len(data)-1; i += 2 {
+	fimg, _ := os.Create("demo/temp/" + e.params.DataId + "_aux_img.txt")
-		data[i], data[i+1] = data[i+1], data[i]
+	defer fimg.Close()
+	fimg.WriteString("字节数 		帧头流水号 		文件号 		帧头时间 	中心辅助数据前4字节(行33-36)\n")
+
+	var startAuxLine bool
+	var preSN uint32
+	for i := 0; i < len(data); {
+		afh := &AuxFrameHead{}
+		afh.Decode(data[i : i+24])
+
+		if !afh.IsValidFrmHead {
+			log.Error("invalid frame head of original raw data", i, len(data), afh.SerialNo, afh.FileNo)
+			return
+		}
+
+		t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000)
+
+		utcTime := binary.BigEndian.Uint32(data[i+32 : i+36])
+		tAux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0)
+
+		startAuxLine = math.Abs(float64(utcTime)-float64(afh.TimeSec)) < 30
+		if startAuxLine {
+			fimg.WriteString(fmt.Sprintf("%d 	%d 		%d 		%s 		%s\n",
+				i,
+				afh.SerialNo,
+				afh.FileNo,
+				t.String(),
+				tAux.String(),
+			))
+
+			if afh.SerialNo-preSN != 16 && preSN != 0 {
+				log.Println("serial number not continuous", afh.SerialNo, preSN)
+			}
+			preSN = afh.SerialNo
+		}
+
+		i += afh.RowLength
 	}
-	return w.Write(data)
+}
+
+// 裁剪图像数据，只保留有效数据
+func (e *Extractor) trimImgRawData(data []byte) []byte {
+	var start, end int
+
+	// 将中心辅助数据时间合理的帧作为第一帧
+	var startAuxLine bool
+	for i := 0; i < len(data); {
+		// 解析帧
+		if i+24 > len(data) {
+			log.Info("length of original image frame head is not engough: ", len(data)-i)
+			break
+		}
+
+		if data[i] == 0xD1 && data[i+1] == 0x5B && data[i+2] == 0xD1 && data[i+3] == 0x5B {
+			log.Debug("find package head: 0xD15BD15B")
+		} else {
+			i++
+			continue
+		}
+
+		afh := &AuxFrameHead{}
+		afh.Decode(data[i : i+24])
+
+		if !afh.IsValidFrmHead {
+			log.Debugf("invalid frame head of original raw data %v", i)
+			i += 1
+			continue
+		}
+
+		utcTime := binary.BigEndian.Uint32(data[i+32 : i+36])               // 相对于2000年的秒数
+		startAuxLine = math.Abs(float64(utcTime)-float64(afh.TimeSec)) < 30 // 时间差小于30秒认为是有效数据
+		if !startAuxLine {
+			i += afh.RowLength
+		} else {
+			start = i
+			nLen := (len(data) - i - 1)
+			nLen = nLen - nLen%(afh.RowLength*16)
+			end = i + nLen
+			fmt.Println("start:", start, "end:", end, "i:", i, "len(data):", len(data),
+				len(data)%(afh.RowLength*16), afh.RowLength,
+			)
+			break
+		}
+	}
+
+	return data[start:end]
 }

extract/envi.go

@@ -50,20 +50,3 @@ func (h *EnviHdr) String() string {
 
 	return buf.String()
 }
-
-func NewBSQWriter(filepath string, content *EnviHdr) (*BSQWriter, error) {
-	fio, err := os.Create(filepath)
-	if err != nil {
-		return nil, err
-	}
-	w := fio
-	return &BSQWriter{filepath, fio, w, content}, nil
-}
-
-func (w *BSQWriter) Write(data []byte) (int, error) {
-	return w.w.Write(data)
-}
-
-func (w *BSQWriter) Close() error {
-	return w.fio.Close()
-}

extract/writer.go

@@ -0,0 +1,69 @@
+package extract
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	EB_AUX         = "EB.AUX"
+	PLAT_AUX       = "PLAT.AUX"
+	PAN_RAW        = "PAN_RAW"
+	MSS_RAW        = "MSS_RAW"
+	PAN_HDR        = "PAN_HDR"
+	MSS_HDR        = "MSS_HDR"
+	MSS_ALL_IN_ONE = "MSS_ALL_IN_ONE"
+)
+
+type l0w struct {
+	w    *bufio.Writer
+	f    *os.File
+	name string
+}
+
+type L0Writer struct {
+	mssStoredType string
+	ws            map[string]*l0w
+}
+
+func newL0Writer(outputDir, name string) *L0Writer {
+	var err error
+	suffix := []string{EB_AUX, PLAT_AUX, PAN_RAW, MSS_RAW, PAN_HDR, MSS_HDR}
+
+	lw := &L0Writer{
+		ws:            make(map[string]*l0w),
+		mssStoredType: MSS_ALL_IN_ONE,
+	}
+
+	for _, s := range suffix {
+		w := &l0w{}
+		w.name = filepath.Join(outputDir, name+"_"+s)
+		w.f, err = os.OpenFile(w.name, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777)
+		if err != nil {
+			logrus.Panic("create file failed: ", w.name, err)
+		}
+		w.w = bufio.NewWriter(w.f)
+		lw.ws[s] = w
+	}
+
+	return lw
+}
+
+func (lw *L0Writer) Close() {
+	for _, w := range lw.ws {
+		w.w.Flush()
+		w.f.Close()
+	}
+}
+
+// 写入 16 位像素值，小端序
+func write16bPixelLittleEndian(w io.Writer, data []byte) (int, error) {
+	for i := 0; i < len(data)-1; i += 2 {
+		data[i], data[i+1] = data[i+1], data[i]
+	}
+	return w.Write(data)
+}