package extract import ( "encoding/binary" "errors" "fmt" "math" "os" "path/filepath" "strings" "time" log "github.com/sirupsen/logrus" ) // 从传输帧文件中分离辅助数据,分别存储到辅助数据文件 _AUX.dat 和图像数据文件 _IMG_{波谱}.dat 中 func (e *Extractor) SeprateAuxAndImgData(dataFile string, segmentIndex int) error { // 打开传输帧文件 - 一次读入内存 data, err := os.ReadFile(dataFile) if err != nil { log.Println("read data from", dataFile, "error:", err.Error()) return err } log.Info("seprate aux and img data from", dataFile) dataLen := len(data) log.Info("length of original data: ", dataLen) var firstFrmHead *AuxFrameHead data, firstFrmHead = e.trimImgRawData(data) dataLen = len(data) log.Info("after trim,length of data: ", dataLen) e.quanlityAnalysis(data) outputDir := e.params.OutputPath name := strings.Join([]string{ e.params.Satellite, "PMS", time.Unix(int64(firstFrmHead.TimeSec)+int64(ReferenceTime2000), int64(firstFrmHead.TimeSecFrac)*1000). Format("20060102_150405"), e.params.DataId, fmt.Sprintf("%03d", firstFrmHead.FileNo), }, "_") lw := newL0Writer(outputDir, name) defer lw.Close() var panEnviHdr, mssEnviHdr EnviHdr var afh AuxFrameHead pandata := make([]byte, 0) msdata := make([][]byte, 4) var header []byte var ebAux []byte var platAux []byte for i := 0; i < dataLen; { if i+4 > dataLen { log.Println("end of data, dataLen:", dataLen, "i:", i) break } // 包头 if data[i] == 0xD1 && data[i+1] == 0x5B && data[i+2] == 0xD1 && data[i+3] == 0x5B { log.Debug("find package head: 0xD15BD15B") } else { i++ continue } // 解析帧 if i+24 > dataLen { log.Info("length of frame head is not engough for frame head") break } afh.Decode(data[i : i+24]) if !afh.IsValidFrmHead { log.Info("invalid frame head of original raw data") i += 1 continue } // 目前只支持线阵模式 if !afh.IsLinerMatrix { log.Error("not liner matrix mode, only support liner matrix mode") break } // 读取一行数据 data[i : i+afh.RowLength] if i+afh.RowLength > dataLen { log.Infof("length of image row data %v is not enough: %v", dataLen-i, afh.RowLength) break } auxTime := binary.BigEndian.Uint32(data[i+32 : i+36]) if math.Abs(float64(auxTime)-float64(afh.TimeSec)) < 30 { header = append(header, data[i:i+24]...) } // 存储辅助数据到临时文件 dataIndex := i + 24 // lw.ws[EB_AUX].w.Write(data[dataIndex : dataIndex+8]) // 8字节焦面电箱辅助数据 ebAux = append(ebAux, data[dataIndex:dataIndex+8]...) dataIndex += 8 // lw.ws[PLAT_AUX].w.Write(data[dataIndex : dataIndex+32]) // 32字节中心机辅助数据 platAux = append(platAux, data[dataIndex:dataIndex+32]...) dataIndex += 32 if !afh.B0 || !afh.B1 || !afh.B2 || !afh.B3 || !afh.B4 { log.Error("not all bands are available") break } // 存储图像数据到临时文件 - 以 ENVI BSQ 格式存储,同时提供 HDR 描述文件 if afh.B0 { // wpan.Write(data[dataIndex : dataIndex+19040]) pandata = append(pandata, data[dataIndex:dataIndex+19040]...) dataIndex += 19040 } if afh.B1 { // write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192]) msdata[0] = append(msdata[0], data[dataIndex:dataIndex+1192]...) dataIndex += 1192 } if afh.B2 { // write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192]) msdata[1] = append(msdata[1], data[dataIndex:dataIndex+1192]...) dataIndex += 1192 } if afh.B3 { // write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192]) msdata[2] = append(msdata[2], data[dataIndex:dataIndex+1192]...) dataIndex += 1192 } if afh.B4 { // write16bPixelLittleEndian(wmss, data[dataIndex:dataIndex+1192]) msdata[3] = append(msdata[3], data[dataIndex:dataIndex+1192]...) dataIndex += 1192 } i = dataIndex // 完成一行数据解析 } if len(msdata[0]) != len(msdata[1]) || len(msdata[2]) != len(msdata[3]) { log.Error("mss data of bands B1-B4 are not equal") return errors.New("mss data of bands B1-B4 are not equal") } // 先判断辅助数据是否完整 // 帧头+辅助数据 if len(header)/24 < len(ebAux)/128 || len(ebAux)/128 != len(platAux)/512 { fmt.Println("aux data length:", len(header)/24, len(ebAux)/128, len(platAux)/512) return errors.New("aux data length is not equal") } auxRows := len(header) / 24 for i := 0; i < auxRows; i++ { lw.ws[AUX].w.Write(header[i*24 : (i+1)*24]) lw.ws[AUX].w.Write(ebAux[i*128 : (i+1)*128]) lw.ws[AUX].w.Write(platAux[i*512 : (i+1)*512]) } _, _, aps, err := e.ExtractAux(lw.ws[AUX].name, lw.ws[AUX].name+".xlsx") if err != nil { return err } auxHeight := len(aps) * 16 // 16行图像数据为一组辅助数据 panHeight := len(pandata) / 19040 mssHeight := 4 * len(msdata[0]) / 4768 log.Println("pan height:", panHeight, "mss height:", mssHeight, "aux height:", auxHeight) // 取最小值作为有效数据长度 efficientHeight := int(math.Min(float64(panHeight), float64(mssHeight))) efficientHeight = int(math.Min(float64(efficientHeight), float64(auxHeight))) log.Println("efficient height:", efficientHeight) // 写入pan数据 for i := 0; i < len(pandata); i += 19040 { start := i + (2+48+38)*2 end := start + 18688 write16bPixelLittleEndian(lw.ws[PAN_RAW].w, pandata[start:end]) panEnviHdr.Lines += 1 if panEnviHdr.Lines >= efficientHeight { break } } mssRowLen := 1192 * 4 for i := 0; i < len(msdata[0]); i += mssRowLen { var err error start := i + (1+11+10)*2 end := start + 4672 _, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[0][start:end]) if err != nil { log.Error("write mss 1 data error:", err.Error()) } _, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[1][start:end]) if err != nil { log.Error("write mss 2 data error:", err.Error()) } _, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[2][start:end]) if err != nil { log.Error("write mss 3 data error:", err.Error()) } _, err = write16bPixelLittleEndian(lw.ws[MSS_RAW].w, msdata[3][start:end]) if err != nil { log.Error("write mss 4 data error:", err.Error()) } mssEnviHdr.Lines += 1 if mssEnviHdr.Lines*4 >= efficientHeight { break } } panEnviHdr.Samples = 9344 panEnviHdr.Bands = 1 lw.ws[PAN_HDR].w.Write([]byte(panEnviHdr.String())) mssEnviHdr.Samples = 2336 * 4 mssEnviHdr.Bands = 1 lw.ws[MSS_HDR].w.Write([]byte(mssEnviHdr.String())) fmt.Println("pan height:", panEnviHdr.Lines, "mss height:", mssEnviHdr.Lines) e.mutex.Lock() defer e.mutex.Unlock() seg := Segment{ Pan: lw.ws[PAN_RAW].name, Mss: lw.ws[MSS_RAW].name, Aux: lw.ws[AUX].name, Id: strings.Split(filepath.Base(lw.ws[AUX].name), ".")[0], StartTime: XMLTime{ TimeZone: "UTC", Value: time.Unix(int64(aps[0].UTCTimeSec)+int64(ReferenceTime2000), int64(aps[0].Microsecond)*1000).UTC().Format(time.RFC3339), }, EndTime: XMLTime{ TimeZone: "UTC", Value: time.Unix(int64(aps[len(aps)-1].UTCTimeSec)+int64(ReferenceTime2000), int64(aps[len(aps)-1].Microsecond)*1000).UTC().Format(time.RFC3339), }, } e.report.Segments = append(e.report.Segments, seg) return nil } func (e *Extractor) quanlityAnalysis(data []byte) { // 数据质量分析 fimg, _ := os.Create("demo/temp/" + e.params.DataId + "/aux.txt") defer fimg.Close() fimg.WriteString("字节数 帧头流水号 文件号 帧头时间 中心辅助数据前4字节(行33-36)\n") var startAuxLine bool var preSN uint32 for i := 0; i < len(data); { afh := &AuxFrameHead{} afh.Decode(data[i : i+24]) if !afh.IsValidFrmHead { log.Errorf("invalid frame head of original raw data i: %v, len: %v", i, len(data)) return } t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), int64(afh.TimeSecFrac)*1000) utcTime := binary.BigEndian.Uint32(data[i+32 : i+36]) tAux := time.Unix(int64(utcTime+uint32(ReferenceTime2000)), 0) startAuxLine = math.Abs(float64(utcTime)-float64(afh.TimeSec)) < 30 if startAuxLine { fimg.WriteString(fmt.Sprintf("%d %d %d %s %s\n", i, afh.SerialNo, afh.FileNo, t.String(), tAux.String(), )) if afh.SerialNo-preSN != 16 && preSN != 0 { log.Println("serial number not continuous", afh.SerialNo, preSN) } preSN = afh.SerialNo } i += afh.RowLength } } // 裁剪图像数据,只保留有效数据 func (e *Extractor) trimImgRawData(data []byte) ([]byte, *AuxFrameHead) { var start, end int afh := &AuxFrameHead{} // 将中心辅助数据时间合理的帧作为第一帧 var startAuxLine bool for i := 0; i < len(data); { // 解析帧 if i+24 > len(data) { log.Info("length of original image frame head is not engough: ", len(data)-i) break } if data[i] == 0xD1 && data[i+1] == 0x5B && data[i+2] == 0xD1 && data[i+3] == 0x5B { log.Debug("find package head: 0xD15BD15B") } else { i++ continue } afh.Decode(data[i : i+24]) if !afh.IsValidFrmHead { log.Debugf("invalid frame head of original raw data %v", i) i += 1 continue } utcTime := binary.BigEndian.Uint32(data[i+32 : i+36]) // 相对于2000年的秒数 startAuxLine = math.Abs(float64(utcTime)-float64(afh.TimeSec)) < 30 // 时间差小于30秒认为是有效数据 if !startAuxLine { i += afh.RowLength } else { start = i nLen := (len(data) - i - 1) nLen = nLen - nLen%(afh.RowLength*16) end = i + nLen break } } return data[start:end], afh } // 9520 16bit 像素 19040 字节,返回有效像素字节起始位置 // 9520 像素 2(Margin) + 48(OB) + 38(Margin) + 9344(Effective Pixels) + 38(Margin) + 48(OB) + 2(Margin) // 7056 像素 2(Margin) + 12(OB) + 18(Margin) + 20(Margin) + 7000(Effective Pixels) + 4(Margin) func PanEffectivePixel(row []byte) (start, end int) { if len(row) < 19040 { return -1, -1 } start = (0 + 2 + 48 + 38) * 2 end = 19040 - (2+48+28)*2 return start, end } // 2384 像素 1+11+10+2336+10+8+8 func MssEffectivePixel(row []byte) (start, end int) { if len(row) < 4768 { return -1, -1 } start = (1 + 11 + 10) * 2 end = 4768 - (10+8+8)*2 return start, end }