Skip to content

Commit

Permalink
ref #15 - ENTSO-E filename convention
Browse files Browse the repository at this point in the history
  • Loading branch information
Christian Biasuzzi committed Jul 19, 2016
1 parent a8af22f commit b31e55d
Show file tree
Hide file tree
Showing 7 changed files with 355 additions and 23 deletions.
31 changes: 30 additions & 1 deletion case-repository/src/main/java/eu/itesla_project/cases/CaseType.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium)
* Copyright (c) 2016, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
Expand All @@ -12,5 +13,33 @@
*/
public enum CaseType {
SN,
FO
FO,
D2,
LT,
RE,
IDCF01,
IDCF02,
IDCF03,
IDCF04,
IDCF05,
IDCF06,
IDCF07,
IDCF08,
IDCF09,
IDCF10,
IDCF11,
IDCF12,
IDCF13,
IDCF14,
IDCF15,
IDCF16,
IDCF17,
IDCF18,
IDCF19,
IDCF20,
IDCF21,
IDCF22,
IDCF23;

}

56 changes: 52 additions & 4 deletions case-repository/src/main/java/eu/itesla_project/cases/EntsoeCaseRepository.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium)
* Copyright (c) 2016, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
Expand Down Expand Up @@ -112,13 +113,33 @@ private static Collection<UcteGeographicalCode> forCountryHacked(Country country
.collect(Collectors.toList());
}

public static boolean isIntraday(CaseType ct) {
return ((ct != null) && (ct.name().startsWith("IDCF")));
}

public static String intraForecastDistanceInHoursSx(CaseType ct) {
return ct.name().substring(4,6);
}

private <R> R scanRepository(DateTime date, CaseType type, Country country, Function<List<ImportContext>, R> handler) {
Collection<UcteGeographicalCode> geographicalCodes = country != null ? forCountryHacked(country)
: Collections.singleton(UcteGeographicalCode.UX);
: Arrays.asList(UcteGeographicalCode.UX, UcteGeographicalCode.UC);

DateTime testDate1=date.minusHours(1);
String typeDirS=type.name();
String typeID=type.name();
if (isIntraday(type)) {
typeDirS = "IDCF";
typeID = intraForecastDistanceInHoursSx(type);
} else if (type.equals(CaseType.D2)) {
typeDirS="2D"; // because enum names cannot be prefixed with a digit
typeID="2D";
}

for (EntsoeFormat format : formats) {
Path formatDir = config.getRootDir().resolve(format.getDirName());
if (Files.exists(formatDir)) {
Path typeDir = formatDir.resolve(type.name());
Path typeDir = formatDir.resolve(typeDirS);
if (Files.exists(typeDir)) {
Path dayDir = typeDir.resolve(String.format("%04d", date.getYear()))
.resolve(String.format("%02d", date.getMonthOfYear()))
Expand All @@ -129,9 +150,12 @@ private <R> R scanRepository(DateTime date, CaseType type, Country country, Func
Collection<String> forbiddenFormats = config.getForbiddenFormatsByGeographicalCode().get(geographicalCode);
if (!forbiddenFormats.contains(format.getImporter().getFormat())) {
for (int i = 9; i >= 0; i--) {
String baseName = String.format("%04d%02d%02d_%02d%02d_" + type + "%01d_" + geographicalCode.name() + "%01d",
String baseName = String.format("%04d%02d%02d_%02d%02d_" + typeID + "%01d_" + geographicalCode.name() + "%01d",
date.getYear(), date.getMonthOfYear(), date.getDayOfMonth(), date.getHourOfDay(), date.getMinuteOfHour(),
date.getDayOfWeek(), i);
if (testDate1.getHourOfDay() == date.getHourOfDay()) {
baseName = baseName.substring(0,9)+'B'+baseName.substring(10);
}
ReadOnlyDataSource ds = dataSourceFactory.create(dayDir, baseName);
if (importContexts == null) {
importContexts = new ArrayList<>();
Expand All @@ -140,6 +164,15 @@ private <R> R scanRepository(DateTime date, CaseType type, Country country, Func
importContexts.add(new ImportContext(format.getImporter(), ds));
}
}
if (importContexts.size()==0 ) { // for info purposes, only
String baseName1 = String.format("%04d%02d%02d_%02d%02d_" + typeID + "%01d_" + geographicalCode.name(),
date.getYear(), date.getMonthOfYear(), date.getDayOfMonth(), date.getHourOfDay(), date.getMinuteOfHour(),
date.getDayOfWeek());
if (testDate1.getHourOfDay() == date.getHourOfDay()) {
baseName1 = baseName1.substring(0,9)+'B'+baseName1.substring(10);
}
LOGGER.warn("could not find any file {}[0-9] in directory {}", baseName1, dayDir);
}
}
}
if (importContexts != null) {
Expand All @@ -148,8 +181,14 @@ private <R> R scanRepository(DateTime date, CaseType type, Country country, Func
return result;
}
}
} else {
LOGGER.warn("could not find any (daydir) directory {}", dayDir);
}
} else {
LOGGER.warn("could not find any (typedir) directory {}", typeDir);
}
} else {
LOGGER.warn("could not find any (formatdir) directory {}", formatDir);
}
}
return null;
Expand Down Expand Up @@ -227,16 +266,25 @@ public Set<DateTime> dataAvailable(CaseType type, Set<Country> countries, Interv
Set<UcteGeographicalCode> geographicalCodes = new HashSet<>();
if (countries == null) {
geographicalCodes.add(UcteGeographicalCode.UX);
geographicalCodes.add(UcteGeographicalCode.UC);
} else {
for (Country country : countries) {
geographicalCodes.addAll(forCountryHacked(country));
}
}
Multimap<DateTime, UcteGeographicalCode> dates = HashMultimap.create();

String typeDirS=type.name();
if (isIntraday(type)) {
typeDirS = "IDCF";
} else if (type.equals(CaseType.D2)) {
typeDirS="2D"; // because enum names cannot be prefixed with a digit
}

for (EntsoeFormat format : formats) {
Path formatDir = config.getRootDir().resolve(format.getDirName());
if (Files.exists(formatDir)) {
Path typeDir = formatDir.resolve(type.name());
Path typeDir = formatDir.resolve(typeDirS);
if (Files.exists(typeDir)) {
browse(typeDir, path -> {
UcteFileName ucteFileName = UcteFileName.parse(path.getFileName().toString());
Expand Down
105 changes: 101 additions & 4 deletions case-repository/src/test/java/eu/itesla_project/cases/EntsoeCaseRepositoryTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium)
* Copyright (c) 2016, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
Expand Down Expand Up @@ -31,9 +32,7 @@
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.*;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
Expand Down Expand Up @@ -152,6 +151,37 @@ public void setUp() throws Exception {
createFile(dir4, "20130115_0015_SN2_D40.uct");
createFile(dir4, "20130115_0015_SN2_D70.uct");
createFile(dir4, "20130115_0015_SN2_D80.uct");

// D2
Path dir5 = fileSystem.getPath("/UCT/2D/2013/01/15");
Files.createDirectories(dir5);
createFile(dir5, "20130115_0030_2D2_FR0.uct");
createFile(dir5, "20130115_0130_2D2_FR0.uct");

// LT
Path dir6 = fileSystem.getPath("/UCT/LT/2013/01/15");
Files.createDirectories(dir6);
createFile(dir6, "20130115_0030_LT2_FR0.uct");
createFile(dir6, "20130115_0130_LT2_FR0.uct");

// RE
Path dir7 = fileSystem.getPath("/UCT/RE/2013/01/15");
Files.createDirectories(dir7);
createFile(dir7, "20130115_0030_RE2_FR0.uct");
createFile(dir7, "20130115_0130_RE2_FR0.uct");

// INTRADAY
Path dir8 = fileSystem.getPath("/UCT/IDCF/2013/01/15");
Files.createDirectories(dir8);
createFile(dir8, "20130115_0330_012_FR0.uct");
createFile(dir8, "20130115_0330_022_FR0.uct");
createFile(dir8, "20130115_0330_032_FR0.uct");

// daylight saving FO
Path dir9 = fileSystem.getPath("/UCT/FO/2016/10/30");
Files.createDirectories(dir9);
createFile(dir9, "20161030_0230_FO7_FR0.uct");
createFile(dir9, "20161030_B230_FO7_FR0.uct");
}

@After
Expand All @@ -169,7 +199,7 @@ public void testLoad() throws Exception {
// check that cim network is loaded instead of uct network
assertTrue(caseRepository.load(DateTime.parse("2013-01-14T00:15:00+01:00"), CaseType.SN, Country.FR).equals(Collections.singletonList(cimNetwork)));

// check that if cim is vorbidden for france, uct is loaded
// check that if cim is forbidden for france, uct is loaded
caseRepository.getConfig().getForbiddenFormatsByGeographicalCode().put(UcteGeographicalCode.FR, "CIM1");
assertTrue(caseRepository.load(DateTime.parse("2013-01-14T00:15:00+01:00"), CaseType.SN, Country.FR).equals(Collections.singletonList(uctNetwork)));

Expand All @@ -193,4 +223,71 @@ public void testDataAvailable() throws Exception {
assertTrue(caseRepository.dataAvailable(CaseType.SN, EnumSet.of(Country.FR), Interval.parse("2013-01-14T00:00:00+01:00/2013-01-14T01:00:00+01:00"))
.equals(Sets.newHashSet(DateTime.parse("2013-01-14T00:15:00+01:00"), DateTime.parse("2013-01-14T00:30:00+01:00"))));
}

@Test
public void testLoadD2() throws Exception {
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:30:00+01:00"), CaseType.D2, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:45:00+01:00"), CaseType.D2, Country.FR).isEmpty());
}

@Test
public void testLoadLT() throws Exception {
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:30:00+01:00"), CaseType.LT, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:45:00+01:00"), CaseType.LT, Country.FR).isEmpty());
}

@Test
public void testLoadRE() throws Exception {
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:30:00+01:00"), CaseType.RE, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:45:00+01:00"), CaseType.RE, Country.FR).isEmpty());
}

@Test
public void testLoadDayLightSaving() throws Exception {
List<Network> networksCEST=caseRepository.load(DateTime.parse("2016-10-30T02:30:00+02:00"), CaseType.FO, Country.FR);
assertTrue(networksCEST.size() == 1);
List<Network> networksCET=caseRepository.load(DateTime.parse("2016-10-30T02:30:00+01:00"), CaseType.FO, Country.FR);
assertTrue(networksCET.size() == 1);
}

@Test
public void testLoadIDCF() throws Exception {
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T03:30:00+01:00"), CaseType.IDCF01, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T03:30:00+01:00"), CaseType.IDCF02, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T03:30:00+01:00"), CaseType.IDCF03, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T03:30:00+01:00"), CaseType.IDCF04, Country.FR).isEmpty());
}

@Test
public void testIsDataAvailable2D() throws Exception {
assertTrue(caseRepository.isDataAvailable(DateTime.parse("2013-01-15T00:30:00+01:00"), CaseType.D2, Country.FR));
}

@Test
public void testDataAvailable2D() throws Exception {
assertTrue(caseRepository.dataAvailable(CaseType.D2, EnumSet.of(Country.FR), Interval.parse("2013-01-15T00:00:00+01:00/2013-01-15T01:30:00+01:00"))
.equals(Sets.newHashSet(DateTime.parse("2013-01-15T00:30:00+01:00"))));
}

@Test
public void testDataAvailableIntraday() throws Exception {
Set<DateTime> dset=caseRepository.dataAvailable(CaseType.IDCF01, EnumSet.of(Country.FR), Interval.parse("2013-01-15T00:00:00+01:00/2013-01-15T05:30:00+01:00"));
System.out.println(dset);
assertTrue(dset.equals(Sets.newHashSet(DateTime.parse("2013-01-15T03:30:00+01:00"))));
}

@Test
public void testDataAvailableDayLightSaving() throws Exception {

// double date CEST + CET
Set<DateTime> dset=caseRepository.dataAvailable(CaseType.FO, EnumSet.of(Country.FR), Interval.parse("2016-10-30T00:00:00+02:00/2016-10-30T03:30:00+01:00"));
System.out.println(dset);
assertTrue(dset.equals(Sets.newHashSet(DateTime.parse("2016-10-30T02:30:00+02:00"),DateTime.parse("2016-10-30T02:30:00+01:00"))));

//just the CET one
dset=caseRepository.dataAvailable(CaseType.FO, EnumSet.of(Country.FR), Interval.parse("2016-10-30T02:30:00+01:00/2016-10-30T03:30:00+01:00"));
System.out.println(dset);
assertTrue(dset.equals(Sets.newHashSet(DateTime.parse("2016-10-30T02:30:00+01:00"))));
}

}
55 changes: 43 additions & 12 deletions ucte-util/src/main/java/eu/itesla_project/ucte/util/UcteFileName.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium)
* Copyright (c) 2016, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
Expand All @@ -8,24 +9,32 @@

import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Interval;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static java.lang.Math.toIntExact;

/**
*
* @author Geoffroy Jamgotchian <geoffroy.jamgotchian at rte-france.com>
*/
public class UcteFileName {

private static final Pattern DATE_REGEX = Pattern.compile(".*(\\d{4})[- /._]?(\\d{2})[- /._]?(\\d{2})[- /._]?(\\d{2})[- /._]?(\\d{2}).*");
private static final Pattern DATE_REGEX = Pattern.compile("(\\d{4})(\\d{2})(\\d{2})_(B\\d|\\d{2})(\\d{2})_(\\w{2})(\\d)_(\\w{2})(\\d).*");

private final DateTime date;

private final int forecastDistance;

private final UcteGeographicalCode geographicalCode;

private static long calcForecastDistance(DateTime date, int offset) {
return new Interval(date.withTimeAtStartOfDay(),date).toDuration().getStandardMinutes() + offset;
}


public static UcteFileName parse(String str) {
DateTime date = DateTime.now();
int forecastDistance = 0;
Expand All @@ -36,21 +45,43 @@ public static UcteFileName parse(String str) {
int year = Integer.parseInt(m.group(1));
int month = Integer.parseInt(m.group(2));
int dayOfMonth = Integer.parseInt(m.group(3));
int hourOfDay = Integer.parseInt(m.group(4));
String hourDayGroup = m.group(4);
boolean cestFlag=hourDayGroup.startsWith("B");
int hourOfDay = (cestFlag==true) ? Integer.parseInt(m.group(4).substring(1)) : Integer.parseInt(m.group(4));
int minute = Integer.parseInt(m.group(5));
date = new DateTime(year, month, dayOfMonth, hourOfDay, minute, DateTimeZone.forID("Europe/Paris"));

// extract horizon and forecast distance
if (str.contains("FO")) {
forecastDistance = 60 * (6 + hourOfDay) + minute; // DACF generated at 18:00 one day ahead7
} else if (str.contains("SN")) {
forecastDistance = 0;
if (cestFlag) {
date = date.plusHours(1);
}
String fileType=m.group(6);
String dweekS=m.group(7);
String cCode=m.group(8);
String versionS=m.group(9);

// extract forecast distance
switch (fileType) {
case "FO" : // Day ahead forecast
forecastDistance = toIntExact(calcForecastDistance(date, 6*60)); // DACF generated at 18:00 one day ahead7
break;
case "2D" : // Two days ahead forecast
forecastDistance = toIntExact(calcForecastDistance(date, 29*60)); // D2CF generated at 19:00 one day ahead7
break;
case "SN" : // Snapshot
forecastDistance = 0;
break;
case "RE" : //Reference //TODO forecastDistance
forecastDistance = 0;
break;
case "LR" : //Long Term Reference //TODO forecastDistance
forecastDistance = 0;
break;
default: //hh Intraday Forecasts, for the rest of the day: two-digits number is the forecast distance, in hours
forecastDistance = Integer.parseInt(m.group(6)) * 60;
break;
}
}
if (str.length() >= 19) {
geographicalCode = UcteGeographicalCode.valueOf(str.substring(18, 20));
}

geographicalCode = UcteGeographicalCode.valueOf(cCode);
}
return new UcteFileName(date, forecastDistance, geographicalCode);
}

Expand Down
Loading

0 comments on commit b31e55d

Please sign in to comment.