From 5b3c4aa0a4ad6154d419ef1042d2b84a960d7d82 Mon Sep 17 00:00:00 2001 From: nuknal Date: Fri, 17 May 2024 23:53:09 +0800 Subject: [PATCH] refactor --- extract/aux.go | 270 ++++++++++++++++++++++------------------------ extract/envi.go | 17 --- extract/writer.go | 69 ++++++++++++ 3 files changed, 196 insertions(+), 160 deletions(-) create mode 100644 extract/writer.go diff --git a/extract/aux.go b/extract/aux.go index e0ed508..61b2059 100644 --- a/extract/aux.go +++ b/extract/aux.go @@ -1,14 +1,12 @@ package extract import ( - "bufio" "encoding/binary" "errors" "fmt" - "io" + "math" "os" "path/filepath" - "sort" "strings" "time" @@ -32,8 +30,8 @@ type AuxFrameHead struct { // B0 全色状态 0：不输出1：输出 ImgMode byte SerialNo uint32 // 当前流水号 - TimeSec uint32 // 时间（秒） 量纲：1us/bit，十六进制无符号整型数 - TimeSecFrac uint32 // 秒小数 + TimeSec uint32 // 时间（秒） + TimeSecFrac uint32 // 秒小数 量纲：1us/bit，十六进制无符号整型数 FileNo uint8 // 文件号 IsValidFrmHead bool @@ -107,130 +105,36 @@ func (afh AuxFrameHead) CheckFrmHead() bool { return false } -type Record struct { - index int - frmHead *AuxFrameHead -} - // 从传输帧文件中分离辅助数据，分别存储到辅助数据文件 _AUX.dat 和图像数据文件 _IMG_{波谱}.dat 中 -func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) error { +func (e *Extractor) SeprateAuxAndImgData(dataFile string, segmentIndex int) error { // 打开传输帧文件 - 一次读入内存 - sData, err := os.ReadFile(sDataFile) + data, err := os.ReadFile(dataFile) if err != nil { - log.Println("read data from", sDataFile, "error:", err.Error()) + log.Println("read data from", dataFile, "error:", err.Error()) return err } - log.Info("seprate aux and img data from", sDataFile) + log.Info("seprate aux and img data from", dataFile) - // 数据质量分析 - fimg, _ := os.Create("demo/temp/" + p.params.DataId + "_aux_img.txt") - defer fimg.Close() - fimg.WriteString("字节数 帧头流水号 文件号 帧头时间 中心辅助数据前4字节(行33-36)\n") - var qmap []*Record - var startAuxLine bool - var preSN uint32 - for i := 0; i < len(sData); { - // 解析帧 - if i+24 > len(sData) { - log.Info("length of original image frame head is not engough: ", len(sData)-i) - break - } - - if sData[i] == 0xD1 && sData[i+1] == 0x5B && sData[i+2] == 0xD1 && sData[i+3] == 0x5B { - log.Debug("find package head: 0xD15BD15B") - } else { - i++ - // log.Println(i,"not find package head: 0xD15BD15B, skip 1 byte") - continue - } - - afh := &AuxFrameHead{} - afh.Decode(sData[i : i+24]) - - if !afh.IsValidFrmHead { - log.Debugf("[%d] invalid frame head of original raw data %v", i, afh.FrmHead) - i += 1 - continue - } - - r := &Record{index: i, frmHead: afh} - qmap = append(qmap, r) - - utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36]) - - t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000) - tAux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0) - - startAuxLine = tAux.Year() == t.Year() && tAux.Month() == t.Month() && tAux.Day() == t.Day() && - tAux.Hour() == t.Hour() && tAux.Minute() == t.Minute() - if !startAuxLine { - // fimg.WriteString(fmt.Sprintf("-----%d not AUX Start -----\n", afh.SerialNo)) - } else { - fimg.WriteString(fmt.Sprintf("%d %d %d %s %s\n", - i, - afh.SerialNo, - afh.FileNo, - t.String(), - tAux.String(), - )) - - if afh.SerialNo-preSN != 16 { - fmt.Println("serial number not continuous", afh.SerialNo, preSN) - } - preSN = afh.SerialNo - } - - i += afh.RowLength - } - - sort.Slice(qmap, func(i, j int) bool { - return qmap[i].index < qmap[j].index - }) - - name := filepath.Base(sDataFile) + name := filepath.Base(dataFile) name = strings.TrimRight(name, filepath.Ext(name)) - outputDir := p.params.OutputPath + outputDir := e.params.OutputPath - aux0 := outputDir + "/" + name + "_EB.AUX" - os.Remove(aux0) - faux0, _ := os.OpenFile(aux0, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777) - waux0 := bufio.NewWriter(faux0) - defer waux0.Flush() - defer faux0.Close() + lw := newL0Writer(outputDir, name) + defer lw.Close() - aux1 := outputDir + "/" + name + "_PLAT.AUX" - os.Remove(aux1) - faux1, _ := os.OpenFile(aux1, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777) - waux1 := bufio.NewWriter(faux1) - defer waux1.Flush() - defer faux1.Close() - - pan := outputDir + "/" + name + "_PAN.RAW" - os.Remove(pan) - fpan, _ := os.OpenFile(pan, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777) - wpan := bufio.NewWriter(fpan) - defer wpan.Flush() - defer fpan.Close() - panEnviHdr := EnviHdr{} - wpanHdr, _ := NewBSQWriter(outputDir+"/"+name+"_PAN.HDR", &panEnviHdr) - defer wpanHdr.Close() - - // 先按单波段存储，再按波段组合存储为 BSQ 格式的 MSS - mss := outputDir + "/" + name + "_MSS.RAW" - os.Remove(mss) - fmss, _ := os.OpenFile(mss, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777) - wmss := bufio.NewWriter(fmss) - defer wmss.Flush() - defer fmss.Close() - mssEnviHdr := EnviHdr{} - wmssHdr, _ := NewBSQWriter(outputDir+"/"+name+"_MSS.HDR", &mssEnviHdr) - defer wmssHdr.Close() + var panEnviHdr, mssEnviHdr EnviHdr var afh AuxFrameHead - dataLen := len(sData) + dataLen := len(data) + log.Info("original data length:", dataLen) + data = e.trimImgRawData(data) + dataLen = len(data) + log.Info("trimmed data length:", dataLen) - mssData := make([][]byte, 4) + e.quanlityAnalysis(data) + + msdata := make([][]byte, 4) var firstLineFound bool var totalLines int var pancount int @@ -241,7 +145,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err } // 包头 - if sData[i] == 0xD1 && sData[i+1] == 0x5B && sData[i+2] == 0xD1 && sData[i+3] == 0x5B { + if data[i] == 0xD1 && data[i+1] == 0x5B && data[i+2] == 0xD1 && data[i+3] == 0x5B { log.Debug("find package head: 0xD15BD15B") } else { i++ @@ -254,7 +158,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err log.Info("length of frame head is not engough for frame head") break } - afh.Decode(sData[i : i+24]) + afh.Decode(data[i : i+24]) // pp.Println(afh) if !afh.IsValidFrmHead { @@ -269,7 +173,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err break } - // 读取一行数据 sData[i : i+afh.RowLength] + // 读取一行数据 data[i : i+afh.RowLength] if i+afh.RowLength > dataLen { log.Infof("length of image row data %v is not enough: %v", dataLen-i, afh.RowLength) break @@ -277,7 +181,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err // 通过中心辅助时间判断是否第一行数据 if !firstLineFound { - utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36]) + utcTime := binary.BigEndian.Uint32(data[i+32 : i+36]) t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000) tAux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0) firstLineFound = tAux.Year() == t.Year() && tAux.Month() == t.Month() && tAux.Day() == t.Day() && @@ -292,39 +196,39 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err // 存储辅助数据到临时文件 dataIndex := i + 24 - waux0.Write(sData[dataIndex : dataIndex+8]) // 8字节焦面电箱辅助数据 + lw.ws[EB_AUX].w.Write(data[dataIndex : dataIndex+8]) // 8字节焦面电箱辅助数据 dataIndex += 8 - waux1.Write(sData[dataIndex : dataIndex+32]) // 32字节中心机辅助数据 + lw.ws[PLAT_AUX].w.Write(data[dataIndex : dataIndex+32]) // 32字节中心机辅助数据 dataIndex += 32 // 存储图像数据到临时文件 - 以 ENVI BSQ 格式存储，同时提供 HDR 描述文件 if afh.B0 { - // wpan.Write(sData[dataIndex : dataIndex+19040]) - write16bPixelLittleEndian(wpan, sData[dataIndex:dataIndex+19040]) + // wpan.Write(data[dataIndex : dataIndex+19040]) + write16bPixelLittleEndian(lw.ws[PAN_RAW].w, data[dataIndex:dataIndex+19040]) dataIndex += 19040 panEnviHdr.Lines += 1 pancount += 19040 } if afh.B1 { - // write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192]) - mssData[0] = append(mssData[0], sData[dataIndex:dataIndex+1192]...) + // write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192]) + msdata[0] = append(msdata[0], data[dataIndex:dataIndex+1192]...) dataIndex += 1192 mssEnviHdr.Lines += 1 } if afh.B2 { - // write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192]) - mssData[1] = append(mssData[1], sData[dataIndex:dataIndex+1192]...) + // write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192]) + msdata[1] = append(msdata[1], data[dataIndex:dataIndex+1192]...) dataIndex += 1192 } if afh.B3 { - // write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192]) - mssData[2] = append(mssData[2], sData[dataIndex:dataIndex+1192]...) + // write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192]) + msdata[2] = append(msdata[2], data[dataIndex:dataIndex+1192]...) dataIndex += 1192 } if afh.B4 { - // write16bPixelLittleEndian(wmss, sData[dataIndex:dataIndex+1192]) - mssData[3] = append(mssData[3], sData[dataIndex:dataIndex+1192]...) + // write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192]) + msdata[3] = append(msdata[3], data[dataIndex:dataIndex+1192]...) dataIndex += 1192 } i = dataIndex // 完成一行数据解析 @@ -332,9 +236,9 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err var bands = 0 for i := 0; i < 4; i++ { - if len(mssData[i]) > 0 { + if len(msdata[i]) > 0 { log.Println("write mss data of band B", i+1) - // _, err := write16bPixelLittleEndian(wmss, mssData[i]) + _, err := write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[i]) if err != nil { log.Error("write mss data error:", err.Error()) } @@ -342,24 +246,104 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string, segmentIndex int) err } } - write16bPixelLittleEndian(wmss, mssData[2]) - panEnviHdr.Samples = 9520 panEnviHdr.Bands = 1 - wpanHdr.Write([]byte(panEnviHdr.String())) + lw.ws[PAN_HDR].w.Write([]byte(panEnviHdr.String())) mssEnviHdr.Lines = mssEnviHdr.Lines / 4 // 多光谱波段分别在 4 行中传输 mssEnviHdr.Samples = 2384 mssEnviHdr.Bands = 1 - wmssHdr.Write([]byte(mssEnviHdr.String())) + lw.ws[MSS_HDR].w.Write([]byte(mssEnviHdr.String())) return nil } -// 写入 16 位像素值，小端序 -func write16bPixelLittleEndian(w io.Writer, data []byte) (int, error) { - for i := 0; i < len(data)-1; i += 2 { - data[i], data[i+1] = data[i+1], data[i] +func (e *Extractor) quanlityAnalysis(data []byte) { + // 数据质量分析 + fimg, _ := os.Create("demo/temp/" + e.params.DataId + "_aux_img.txt") + defer fimg.Close() + fimg.WriteString("字节数 帧头流水号 文件号 帧头时间 中心辅助数据前4字节(行33-36)\n") + + var startAuxLine bool + var preSN uint32 + for i := 0; i < len(data); { + afh := &AuxFrameHead{} + afh.Decode(data[i : i+24]) + + if !afh.IsValidFrmHead { + log.Error("invalid frame head of original raw data", i, len(data), afh.SerialNo, afh.FileNo) + return + } + + t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000) + + utcTime := binary.BigEndian.Uint32(data[i+32 : i+36]) + tAux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0) + + startAuxLine = math.Abs(float64(utcTime)-float64(afh.TimeSec)) < 30 + if startAuxLine { + fimg.WriteString(fmt.Sprintf("%d %d %d %s %s\n", + i, + afh.SerialNo, + afh.FileNo, + t.String(), + tAux.String(), + )) + + if afh.SerialNo-preSN != 16 && preSN != 0 { + log.Println("serial number not continuous", afh.SerialNo, preSN) + } + preSN = afh.SerialNo + } + + i += afh.RowLength } - return w.Write(data) +} + +// 裁剪图像数据，只保留有效数据 +func (e *Extractor) trimImgRawData(data []byte) []byte { + var start, end int + + // 将中心辅助数据时间合理的帧作为第一帧 + var startAuxLine bool + for i := 0; i < len(data); { + // 解析帧 + if i+24 > len(data) { + log.Info("length of original image frame head is not engough: ", len(data)-i) + break + } + + if data[i] == 0xD1 && data[i+1] == 0x5B && data[i+2] == 0xD1 && data[i+3] == 0x5B { + log.Debug("find package head: 0xD15BD15B") + } else { + i++ + continue + } + + afh := &AuxFrameHead{} + afh.Decode(data[i : i+24]) + + if !afh.IsValidFrmHead { + log.Debugf("invalid frame head of original raw data %v", i) + i += 1 + continue + } + + utcTime := binary.BigEndian.Uint32(data[i+32 : i+36]) // 相对于2000年的秒数 + startAuxLine = math.Abs(float64(utcTime)-float64(afh.TimeSec)) < 30 // 时间差小于30秒认为是有效数据 + if !startAuxLine { + i += afh.RowLength + } else { + start = i + nLen := (len(data) - i - 1) + nLen = nLen - nLen%(afh.RowLength*16) + end = i + nLen + fmt.Println("start:", start, "end:", end, "i:", i, "len(data):", len(data), + len(data)%(afh.RowLength*16), afh.RowLength, + ) + break + } + } + + return data[start:end] } diff --git a/extract/envi.go b/extract/envi.go index 406b41f..5811506 100644 --- a/extract/envi.go +++ b/extract/envi.go @@ -50,20 +50,3 @@ func (h *EnviHdr) String() string { return buf.String() } - -func NewBSQWriter(filepath string, content *EnviHdr) (*BSQWriter, error) { - fio, err := os.Create(filepath) - if err != nil { - return nil, err - } - w := fio - return &BSQWriter{filepath, fio, w, content}, nil -} - -func (w *BSQWriter) Write(data []byte) (int, error) { - return w.w.Write(data) -} - -func (w *BSQWriter) Close() error { - return w.fio.Close() -} diff --git a/extract/writer.go b/extract/writer.go new file mode 100644 index 0000000..c79701b --- /dev/null +++ b/extract/writer.go @@ -0,0 +1,69 @@ +package extract + +import ( + "bufio" + "io" + "os" + "path/filepath" + + "github.com/sirupsen/logrus" +) + +const ( + EB_AUX = "EB.AUX" + PLAT_AUX = "PLAT.AUX" + PAN_RAW = "PAN_RAW" + MSS_RAW = "MSS_RAW" + PAN_HDR = "PAN_HDR" + MSS_HDR = "MSS_HDR" + MSS_ALL_IN_ONE = "MSS_ALL_IN_ONE" +) + +type l0w struct { + w *bufio.Writer + f *os.File + name string +} + +type L0Writer struct { + mssStoredType string + ws map[string]*l0w +} + +func newL0Writer(outputDir, name string) *L0Writer { + var err error + suffix := []string{EB_AUX, PLAT_AUX, PAN_RAW, MSS_RAW, PAN_HDR, MSS_HDR} + + lw := &L0Writer{ + ws: make(map[string]*l0w), + mssStoredType: MSS_ALL_IN_ONE, + } + + for _, s := range suffix { + w := &l0w{} + w.name = filepath.Join(outputDir, name+"_"+s) + w.f, err = os.OpenFile(w.name, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0777) + if err != nil { + logrus.Panic("create file failed: ", w.name, err) + } + w.w = bufio.NewWriter(w.f) + lw.ws[s] = w + } + + return lw +} + +func (lw *L0Writer) Close() { + for _, w := range lw.ws { + w.w.Flush() + w.f.Close() + } +} + +// 写入 16 位像素值，小端序 +func write16bPixelLittleEndian(w io.Writer, data []byte) (int, error) { + for i := 0; i < len(data)-1; i += 2 { + data[i], data[i+1] = data[i+1], data[i] + } + return w.Write(data) +}