解析气象Grib文件实例 girbjava
程序员文章站
2024-03-15 13:27:53
...
GRIB 码是与计算机无关的压缩的二进制编码,主要用来表示数值天气预报的产品资料。现行的GRIB 码版本有GRIB1 和GRIB2 两种格式。 GRIB2较之GRIB1具有加大优点而被广泛使用。如:表示多维数据、模块性结构、支持多种压缩方式、IEEE标准浮点表示法等。
首先需要引入的三方库
Maven
通过读取grib 文件 来解析数据
Options :
FloatValue java
GribRecord
Grib2Json
因为grib 数据格式是这样的 以时间的方式保存一个格点所有数据
而 mongoDB 的保存方式以坐标为主的保存方式所以 在业务逻辑上想找坐标对应所有时间的数据 然后依次坐标遍历所有当前坐标的时间数据
首先需要引入的三方库
Maven
<repositories> <repository> <id>unidata</id> <name>THREDDS</name> <url>https://artifacts.unidata.ucar.edu/content/repositories/unidata-releases/</url> </repositories> <dependency> <groupId>joda-time</groupId> <artifactId>joda-time</artifactId> <version>2.3</version> </dependency> <dependency> <groupId>org.glassfish</groupId> <artifactId>javax.json</artifactId> <version>1.0.3</version> </dependency> <dependency> <groupId>com.lexicalscope.jewelcli</groupId> <artifactId>jewelcli</artifactId> <version>0.8.8</version> </dependency> <dependency> <groupId>edu.ucar</groupId> <artifactId>grib</artifactId> <version>4.3.19</version> </dependency>
通过读取grib 文件 来解析数据
public void fileData() { try{ //grib 文件夹里包含N个grib 文件 File folder = new File(environment.getProperty("weatherFolder")); File[] files = folder.listFiles(); for (File file : files) { String[] args = {"--data","--names",file.getPath()}; verify(args); } }catch(Exception e){ log.error("文件导入发生异常:{}",e); } }
public void verify(String[] args) { Options options = CliFactory.parseArguments(Options.class, args); try { if (!options.getFile().exists()) { log.error("Cannot find input file {0}",options.getFile()); return; } if(options.getFile().isDirectory()){ return; } log.info("读取 {} 气象文件",options.getFile().getName()); new Grib2Json(options.getFile(), options).read(table); }catch (Exception e) { log.error("verify error:{}",e); e.printStackTrace(); System.exit(1); } }
Options :
package com.kedalo.databus.grib2; import com.lexicalscope.jewel.cli.*; import java.io.File; /** * @date 2019-09-17 * * * @author liyulong * 该功能实现一部分 后续可扩展 */ @CommandLineInterface(application="grib2json", order=OptionOrder.LONGNAME) public interface Options { @Option(longName="help", shortName="h", description="帮助命令") boolean getShowHelp(); @Option(longName="names", shortName="n", description="打印返回字段名称") boolean getPrintNames(); @Option(longName="data", shortName="d", description="打印数据字段") boolean getPrintData(); @Option(longName="compact", shortName="c", description="转化为JSON格式") boolean isCompactFormat(); @Option(longName="verbose", shortName="v", description="启动日志记录") boolean getEnableLogging(); @Option( longName="output", shortName="o", description="输出命令 例:-o 文件路径", defaultToNull=true) File getOutput(); @Unparsed(name="FILE", defaultToNull=true) File getFile(); @Option( longName={"filter.discipline", "fd"}, description="行业过滤起 暂时无用", defaultToNull=true) Integer getFilterDiscipline(); @Option( longName={"filter.category", "fc"}, description="类型过滤器 暂时无用", defaultToNull=true) Integer getFilterCategory(); @Option( longName={"filter.parameter", "fp"}, description="参数过滤器 暂时无用", defaultToNull=true) String getFilterParameter(); @Option( longName={"filter.surface", "fs"}, description="surface 字段暂时无用", defaultToNull=true) Integer getFilterSurface(); @Option( longName={"filter.value", "fv"}, description="过滤 surface 内容 暂时无用", defaultToNull=true) Double getFilterValue(); }
FloatValue java
package com.kedalo.databus.grib2; import javax.json.JsonNumber; import java.math.BigDecimal; import java.math.BigInteger; /** * 2014-01-17 * @author liyulong */ final class FloatValue implements JsonNumber { private final float value; private BigDecimal bd; FloatValue(float value) { this.value = value; } @Override public ValueType getValueType() { return ValueType.NUMBER; } @Override public String toString() { if (Float.isNaN(value)) { return "\"NaN\""; } else if (value == Float.POSITIVE_INFINITY) { return "\"-Infinity\""; } else if (value == Float.NEGATIVE_INFINITY) { return "\"Infinity\""; } else { return Float.toString(value); } } @Override public boolean isIntegral() { return bigDecimalValue().scale() == 0; } @Override public int intValue() { return (int)value; } @Override public int intValueExact() { return bigDecimalValue().intValueExact(); } @Override public long longValue() { return (long)value; } @Override public long longValueExact() { return bigDecimalValue().longValueExact(); } @Override public BigInteger bigIntegerValue() { return bigDecimalValue().toBigInteger(); } @Override public BigInteger bigIntegerValueExact() { return bigDecimalValue().toBigIntegerExact(); } @Override public double doubleValue() { return (double)value; } @Override public BigDecimal bigDecimalValue() { return bd != null ? bd : (bd = new BigDecimal(value)); } @Override public boolean equals(Object that) { return that instanceof JsonNumber && this.bigDecimalValue().equals(((JsonNumber)that).bigDecimalValue()); } @Override public int hashCode() { return bigDecimalValue().hashCode(); } }
GribRecord
package com.kedalo.databus.grib2; import static ucar.grib.GribNumbers.BIT_5; import static ucar.grib.GribNumbers.UNDEFINED; import static ucar.grib.GribNumbers.isBitSet; import static ucar.grib.grib2.Grib2Tables.codeTable3_1; import static ucar.grib.grib2.Grib2Tables.codeTable3_2; import static ucar.grib.grib2.Grib2Tables.codeTable4_0; import static ucar.grib.grib2.Grib2Tables.codeTable4_3; import static ucar.grib.grib2.Grib2Tables.codeTable4_5; import static ucar.grib.grib2.ParameterTable.getCategoryName; import static ucar.grib.grib2.ParameterTable.getParameterName; import static ucar.grib.grib2.ParameterTable.getParameterUnit; import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import com.kedalo.databus.bean.GribWeather; import com.kedalo.databus.util.WeatherUtil; import ucar.grib.grib2.Grib2Data; import ucar.grib.grib2.Grib2GDSVariables; import ucar.grib.grib2.Grib2IdentificationSection; import ucar.grib.grib2.Grib2IndicatorSection; import ucar.grib.grib2.Grib2Pds; import ucar.grib.grib2.Grib2Record; /** * @author liyulong * @date 2019-09-17 * grib 保存 * */ public class GribRecord { private final Grib2Record record; private final Grib2IndicatorSection ins; private final Options options; private final Grib2IdentificationSection ids; private final Grib2Pds pds; private final Grib2GDSVariables gds; GribRecord(Grib2Record record, Options options) { this.record = record; this.options = options; this.ins = record.getIs(); this.ids = record.getId(); this.pds = record.getPDS().getPdsVars(); this.gds = record.getGDS().getGdsVars(); } public Map<String,Object> getHead(){ int productDef = pds.getProductDefinitionTemplate(); int discipline = ins.getDiscipline(); int paramCategory = pds.getParameterCategory(); int paramNumber = pds.getParameterNumber(); int gridTemplate = gds.getGdtn(); Map<String,Object> recordData = new HashMap<String,Object>(); //学科 recordData.put("discipline", ins.getDiscipline()); recordData.put("disciplineName", ins.getDisciplineName()); //Grib recordData.put("gribEdition", ins.getGribEdition()); //中心点 固定值38 recordData.put("center", ids.getCenter_id()); recordData.put("centerName",WeatherUtil.codeToStr(ids.getCenter_id())); //子中心点 recordData.put("subcenter", ids.getSubcenter_id()); recordData.put("timeLong", ids.getRefTime()); recordData.put("refTime", new Date(ids.getRefTime())); // recordData.put("refTime", new DateTime(ids.getRefTime()).toDate()); recordData.put("time", WeatherUtil.addHourTime(new DateTime(ids.getRefTime()).toDate(), pds.getForecastTime())); recordData.put("significanceOfRT", ids.getSignificanceOfRT()); recordData.put("significanceOfRTName", ids.getSignificanceOfRTName()); recordData.put("productStatus", ids.getProductStatus()); recordData.put("productStatusName", ids.getProductStatusName()); recordData.put("productType", ids.getProductType()); recordData.put("productTypeName", ids.getProductStatusName()); recordData.put("productDefinitionTemplate", productDef); recordData.put("productDefinitionTemplateName", codeTable4_0(productDef)); recordData.put("parameterCategory", paramCategory); recordData.put("parameterCategoryName",getCategoryName(discipline, paramCategory)); recordData.put("parameterNumber", paramNumber); recordData.put("parameterNumberName",getParameterName(discipline, paramCategory, paramNumber)); recordData.put("parameterUnit", getParameterUnit(discipline, paramCategory, paramNumber)); recordData.put("genProcessType", pds.getGenProcessType()); recordData.put("genProcessTypeName", codeTable4_3(pds.getGenProcessType())); recordData.put("forecastTime", pds.getForecastTime()); recordData.put("surface1Type", pds.getLevelType1()); recordData.put("surface1TypeName", codeTable4_5(pds.getLevelType1())); recordData.put("surface1Value", pds.getLevelValue1()); recordData.put("surface2Type", pds.getLevelType2()); recordData.put("surface2TypeName",codeTable4_5(pds.getLevelType2())); recordData.put("surface2Value", pds.getLevelValue2()); recordData.put("gridDefinitionTemplate", gridTemplate); recordData.put("gridDefinitionTemplateName",codeTable3_1(gridTemplate)); recordData.put("numberPoints", gds.getNumberPoints()); switch (gridTemplate) { case 0: // Template 3.0 case 1: // Template 3.1 case 2: // Template 3.2 case 3: // Template 3.3 writeLonLatGrid(recordData); break; case 10: // Template 3.10 writeMercatorGrid(recordData); break; case 20: // Template 3.20 writePolarStereographicGrid(recordData); break; case 30: // Template 3.30 writeLambertConformalGrid(recordData); break; case 40: // Template 3.40 case 41: // Template 3.41 case 42: // Template 3.42 case 43: // Template 3.43 writeLonLatGrid(recordData); break; case 90: // Template 3.90 writeSpaceOrOrthographicGrid(recordData); break; case 204: // Template 3.204 writeCurvilinearGrid(recordData); break; } return recordData; } private void writeCurvilinearGrid(Map<String,Object> recordData) { writeGridShape(recordData); writeGridSize(recordData); } private void writeSpaceOrOrthographicGrid(Map<String,Object> recordData) { writeGridShape(recordData); writeGridSize(recordData); writeAngle(recordData); writeLonLatBounds(recordData); recordData.put("lop", gds.getLop()); recordData.put("lap", gds.getLap()); recordData.put("xp", gds.getXp()); recordData.put("yp", gds.getYp()); recordData.put("nr", gds.getNr()); recordData.put("xo", gds.getXo()); recordData.put("yo", gds.getYo()); } private void writeGridShape(Map<String,Object> recordData) { recordData.put("shape",codeTable3_2(gds.getShape())); recordData.put("shapeName", codeTable3_2(gds.getShape())); switch (gds.getShape()) { case 1: recordData.put("earthRadius", gds.getEarthRadius()); break; case 3: recordData.put("majorAxis", gds.getMajorAxis()); recordData.put("minorAxis", gds.getMinorAxis()); break; } } private void writeGridSize(Map<String,Object> recordData) { recordData.put("gridUnits", gds.getGridUnits()); recordData.put("resolution", gds.getResolution()); recordData.put("winds", isBitSet(gds.getResolution(), BIT_5) ? "relative" : "true"); recordData.put("scanMode", gds.getScanMode()); recordData.put("nx", gds.getNx()); recordData.put("ny", gds.getNy()); } private void writeAngle(Map<String,Object> recordData) { recordData.put("angle", gds.getAngle()); recordData.put("basicAngle", gds.getBasicAngle()); recordData.put("subDivisions", gds.getSubDivisions()); } private void putIfSet(Map<String,Object> recordData,String key,float value){ if ( value != UNDEFINED) { recordData.put(key, value); } } private void writeLonLatBounds(Map<String,Object> recordData) { putIfSet(recordData,"lo1",gds.getLo1()); putIfSet(recordData,"la1",gds.getLa1()); putIfSet(recordData,"lo2",gds.getLo2()); putIfSet(recordData,"la2",gds.getLa2()); putIfSet(recordData,"dx",gds.getDx()); putIfSet(recordData,"dy",gds.getDy()); } private void writeRotationAndStretch(Map<String,Object> recordData) { putIfSet(recordData,"spLon", gds.getSpLon()); putIfSet(recordData,"spLat", gds.getSpLat()); putIfSet(recordData,"rotationAngle", gds.getRotationAngle()); putIfSet(recordData,"poleLon", gds.getPoleLon()); putIfSet(recordData,"poleLat", gds.getPoleLat()); putIfSet(recordData,"stretchingFactor", gds.getStretchingFactor()); } private void writeLonLatGrid(Map<String,Object> recordData) { writeGridShape(recordData); writeGridSize(recordData); writeAngle(recordData); writeLonLatBounds(recordData); writeRotationAndStretch(recordData); putIfSet(recordData,"np", gds.getNp()); } private void writeMercatorGrid(Map<String,Object> recordData) { writeGridShape(recordData); writeGridSize(recordData); writeAngle(recordData); writeLonLatBounds(recordData); } private void writePolarStereographicGrid(Map<String,Object> recordData) { writeGridShape(recordData); writeGridSize(recordData); writeLonLatBounds(recordData); } private void writeLambertConformalGrid(Map<String,Object> recordData) { writeGridShape(recordData); writeGridSize(recordData); writeLonLatBounds(recordData); writeRotationAndStretch(recordData); recordData.put("laD", gds.getLaD()); recordData.put("loV", gds.getLoV()); recordData.put("projectionFlag", gds.getProjectionFlag()); recordData.put("latin1", gds.getLatin1()); recordData.put("latin2", gds.getLatin2()); } List<FloatValue> readData(Grib2Data gd) throws IOException { List<FloatValue> list = new ArrayList<FloatValue>(); float[] data = gd.getData(record.getGdsOffset(), record.getPdsOffset(), ids.getRefTime()); if (data != null) { for (float value : data) { list.add(new FloatValue(value)); } } return list; } }
Grib2Json
因为grib 数据格式是这样的 以时间的方式保存一个格点所有数据
而 mongoDB 的保存方式以坐标为主的保存方式所以 在业务逻辑上想找坐标对应所有时间的数据 然后依次坐标遍历所有当前坐标的时间数据
package com.kedalo.databus.grib2; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.kedalo.databus.comm.MongoClientCommon; import com.kedalo.databus.content.SpringContext; import com.kedalo.databus.util.WeatherUtil; import ucar.grib.grib2.*; import ucar.unidata.io.RandomAccessFile; import java.io.*; import java.util.*; /** * @author liyulong * Date 2019-09-17 * 解析Grib文件 */ public final class Grib2Json{ private static final Logger log = LoggerFactory.getLogger(Grib2Json.class); private final File file; private final Options option; MongoClientCommon mongoClient = (MongoClientCommon) SpringContext.getBean(MongoClientCommon.class); public Grib2Json(File file, Options option) { if (!file.exists()) { log.error("Cannot find input file {0}",file); throw new IllegalArgumentException("Cannot find input file: " + file); } this.file = file; this.option = option; } /*** 这里是处理业务方法 这里是将数据解析出来存入MongoDB 因为业务原因 grib 保存结构和业务平台需要的结构相反 一个是扁平是存储 一个是 树形存储 所以 业务逻辑有点绕,请自行忽略 */ private void read(RandomAccessFile raf, Grib2Input input, Options options,String table) throws IOException { List<Grib2Record> records = input.getRecords(); float startLon = 0;//开始经度 float startLat = 0;//开始纬度 float x = 0;//纬度每次变化的值 float y = 0;//经度每次变化的值 float endLon=0;//结束的经度 float endLat=0;//结束的纬度 float[] data = null;//获取data个数 int forecast = 0; if(records.size() > 0){ Grib2Record temp = records.get(0); GribRecord rw = new GribRecord(temp, options); Map<String,Object> map = rw.getHead(); forecast = (int)map.get("forecastTime"); startLon = WeatherUtil.toFloat((float)map.get("lo1")); endLon = WeatherUtil.toFloat((float) map.get("lo2")); x = WeatherUtil.toFloat((float) map.get("dx")); y = WeatherUtil.toFloat((float) map.get("dy")); startLat = WeatherUtil.toFloat((float) map.get("la1")); endLat = WeatherUtil.toFloat((float) map.get("la2")); data = new Grib2Data(raf).getData(temp.getGdsOffset(), temp.getPdsOffset(), temp.getId().getRefTime()); } int lonCount = WeatherUtil.scale(WeatherUtil.sub(endLon, startLon)/y, 0).intValue(); int latCount = WeatherUtil.scale(WeatherUtil.sub(endLat, startLat)/x, 0).intValue(); float tempLon = 0; float tempLat = 0; int count = 0; for (int i = 0; i < data.length; i++) { tempLon = startLon + (y*(i%(lonCount+1))); tempLat = startLat +(x*count%latCount); count = i/(lonCount+1); Document doc = new Document(); Map<String, Object> documentMap = new HashMap<String, Object>(); documentMap.put("type", "Point"); List<Float> list = new ArrayList<Float>(); list.add(tempLon); list.add(tempLat); documentMap.put("coordinates",list); doc.append("loc",documentMap); list = new ArrayList<Float>(); for (int j = 0; j < records.size(); j++) { Grib2Record temp = records.get(j); GribRecord rw = new GribRecord(temp, options); Map<String,Object> map = rw.getHead(); doc.append("forecastTime",forecast) .append("parameterNumber", map.get("parameterNumber")) .append("parameterNumberName", map.get("parameterNumberName")) .append("parameterCategory", map.get("parameterCategory")) .append("parameterCategoryName", map.get("parameterCategoryName")) .append("parameterUnit", map.get("parameterUnit")) .append("refTime", map.get("refTime")); data = new Grib2Data(raf).getData(temp.getGdsOffset(), temp.getPdsOffset(), temp.getId().getRefTime()); list.add(data[i]); } doc.append("data", list); //保存至MongoDB mongoClient.insertOne(table,doc); } mongoClient.createIndex(table, "loc", "2dsphere"); } public void read(String table) throws IOException { RandomAccessFile raf = new RandomAccessFile(file.getPath(), "r"); raf.order(RandomAccessFile.BIG_ENDIAN); Grib2Input input = new Grib2Input(raf); input.scan(false, false); read(raf, input, option,table); } }