Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Entsoe filename convention #37

Closed
wants to merge 3 commits into from
Closed

Entsoe filename convention #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b31e55d
ref #15 - ENTSO-E filename convention
Jul 19, 2016
ba6de80
Merge branch 'master' into entsoe_filename_convention
Jul 26, 2016
a286c45
renames class UcteFileName to EntsoeFileName
Jul 26, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 30 additions & 1 deletion case-repository/src/main/java/eu/itesla_project/cases/CaseType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium)
* Copyright (c) 2016, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
Expand All @@ -12,5 +13,33 @@
*/
public enum CaseType {
SN,
FO
FO,
D2,
LT,
RE,
IDCF01,
IDCF02,
IDCF03,
IDCF04,
IDCF05,
IDCF06,
IDCF07,
IDCF08,
IDCF09,
IDCF10,
IDCF11,
IDCF12,
IDCF13,
IDCF14,
IDCF15,
IDCF16,
IDCF17,
IDCF18,
IDCF19,
IDCF20,
IDCF21,
IDCF22,
IDCF23;

}

66 changes: 57 additions & 9 deletions case-repository/src/main/java/eu/itesla_project/cases/EntsoeCaseRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium)
* Copyright (c) 2016, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
Expand All @@ -16,7 +17,7 @@
import eu.itesla_project.iidm.import_.Importers;
import eu.itesla_project.iidm.network.Country;
import eu.itesla_project.iidm.network.Network;
import eu.itesla_project.ucte.util.UcteFileName;
import eu.itesla_project.ucte.util.EntsoeFileName;
import eu.itesla_project.ucte.util.UcteGeographicalCode;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
Expand Down Expand Up @@ -112,13 +113,33 @@ private static Collection<UcteGeographicalCode> forCountryHacked(Country country
.collect(Collectors.toList());
}

public static boolean isIntraday(CaseType ct) {
return ((ct != null) && (ct.name().startsWith("IDCF")));
}

public static String intraForecastDistanceInHoursSx(CaseType ct) {
return ct.name().substring(4,6);
}

private <R> R scanRepository(DateTime date, CaseType type, Country country, Function<List<ImportContext>, R> handler) {
Collection<UcteGeographicalCode> geographicalCodes = country != null ? forCountryHacked(country)
: Collections.singleton(UcteGeographicalCode.UX);
: Arrays.asList(UcteGeographicalCode.UX, UcteGeographicalCode.UC);

DateTime testDate1=date.minusHours(1);
String typeDirS=type.name();
String typeID=type.name();
if (isIntraday(type)) {
typeDirS = "IDCF";
typeID = intraForecastDistanceInHoursSx(type);
} else if (type.equals(CaseType.D2)) {
typeDirS="2D"; // because enum names cannot be prefixed with a digit
typeID="2D";
}

for (EntsoeFormat format : formats) {
Path formatDir = config.getRootDir().resolve(format.getDirName());
if (Files.exists(formatDir)) {
Path typeDir = formatDir.resolve(type.name());
Path typeDir = formatDir.resolve(typeDirS);
if (Files.exists(typeDir)) {
Path dayDir = typeDir.resolve(String.format("%04d", date.getYear()))
.resolve(String.format("%02d", date.getMonthOfYear()))
Expand All @@ -129,9 +150,12 @@ private <R> R scanRepository(DateTime date, CaseType type, Country country, Func
Collection<String> forbiddenFormats = config.getForbiddenFormatsByGeographicalCode().get(geographicalCode);
if (!forbiddenFormats.contains(format.getImporter().getFormat())) {
for (int i = 9; i >= 0; i--) {
String baseName = String.format("%04d%02d%02d_%02d%02d_" + type + "%01d_" + geographicalCode.name() + "%01d",
String baseName = String.format("%04d%02d%02d_%02d%02d_" + typeID + "%01d_" + geographicalCode.name() + "%01d",
date.getYear(), date.getMonthOfYear(), date.getDayOfMonth(), date.getHourOfDay(), date.getMinuteOfHour(),
date.getDayOfWeek(), i);
if (testDate1.getHourOfDay() == date.getHourOfDay()) {
baseName = baseName.substring(0,9)+'B'+baseName.substring(10);
}
ReadOnlyDataSource ds = dataSourceFactory.create(dayDir, baseName);
if (importContexts == null) {
importContexts = new ArrayList<>();
Expand All @@ -140,6 +164,15 @@ private <R> R scanRepository(DateTime date, CaseType type, Country country, Func
importContexts.add(new ImportContext(format.getImporter(), ds));
}
}
if (importContexts.size()==0 ) { // for info purposes, only
String baseName1 = String.format("%04d%02d%02d_%02d%02d_" + typeID + "%01d_" + geographicalCode.name(),
date.getYear(), date.getMonthOfYear(), date.getDayOfMonth(), date.getHourOfDay(), date.getMinuteOfHour(),
date.getDayOfWeek());
if (testDate1.getHourOfDay() == date.getHourOfDay()) {
baseName1 = baseName1.substring(0,9)+'B'+baseName1.substring(10);
}
LOGGER.warn("could not find any file {}[0-9] in directory {}", baseName1, dayDir);
}
}
}
if (importContexts != null) {
Expand All @@ -148,8 +181,14 @@ private <R> R scanRepository(DateTime date, CaseType type, Country country, Func
return result;
}
}
} else {
LOGGER.warn("could not find any (daydir) directory {}", dayDir);
}
} else {
LOGGER.warn("could not find any (typedir) directory {}", typeDir);
}
} else {
LOGGER.warn("could not find any (formatdir) directory {}", formatDir);
}
}
return null;
Expand Down Expand Up @@ -227,24 +266,33 @@ public Set<DateTime> dataAvailable(CaseType type, Set<Country> countries, Interv
Set<UcteGeographicalCode> geographicalCodes = new HashSet<>();
if (countries == null) {
geographicalCodes.add(UcteGeographicalCode.UX);
geographicalCodes.add(UcteGeographicalCode.UC);
} else {
for (Country country : countries) {
geographicalCodes.addAll(forCountryHacked(country));
}
}
Multimap<DateTime, UcteGeographicalCode> dates = HashMultimap.create();

String typeDirS=type.name();
if (isIntraday(type)) {
typeDirS = "IDCF";
} else if (type.equals(CaseType.D2)) {
typeDirS="2D"; // because enum names cannot be prefixed with a digit
}

for (EntsoeFormat format : formats) {
Path formatDir = config.getRootDir().resolve(format.getDirName());
if (Files.exists(formatDir)) {
Path typeDir = formatDir.resolve(type.name());
Path typeDir = formatDir.resolve(typeDirS);
if (Files.exists(typeDir)) {
browse(typeDir, path -> {
UcteFileName ucteFileName = UcteFileName.parse(path.getFileName().toString());
UcteGeographicalCode geographicalCode = ucteFileName.getGeographicalCode();
EntsoeFileName entsoeFileName = EntsoeFileName.parse(path.getFileName().toString());
UcteGeographicalCode geographicalCode = entsoeFileName.getGeographicalCode();
if (geographicalCode != null
&& !config.getForbiddenFormatsByGeographicalCode().get(geographicalCode).contains(format.getImporter().getFormat())
&& interval.contains(ucteFileName.getDate())) {
dates.put(ucteFileName.getDate(), geographicalCode);
&& interval.contains(entsoeFileName.getDate())) {
dates.put(entsoeFileName.getDate(), geographicalCode);
}
});
}
Expand Down
105 changes: 101 additions & 4 deletions case-repository/src/test/java/eu/itesla_project/cases/EntsoeCaseRepositoryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* Copyright (c) 2016, All partners of the iTesla project (http://www.itesla-project.eu/consortium)
* Copyright (c) 2016, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
Expand Down Expand Up @@ -31,9 +32,7 @@
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.*;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
Expand Down Expand Up @@ -152,6 +151,37 @@ public void setUp() throws Exception {
createFile(dir4, "20130115_0015_SN2_D40.uct");
createFile(dir4, "20130115_0015_SN2_D70.uct");
createFile(dir4, "20130115_0015_SN2_D80.uct");

// D2
Path dir5 = fileSystem.getPath("/UCT/2D/2013/01/15");
Files.createDirectories(dir5);
createFile(dir5, "20130115_0030_2D2_FR0.uct");
createFile(dir5, "20130115_0130_2D2_FR0.uct");

// LT
Path dir6 = fileSystem.getPath("/UCT/LT/2013/01/15");
Files.createDirectories(dir6);
createFile(dir6, "20130115_0030_LT2_FR0.uct");
createFile(dir6, "20130115_0130_LT2_FR0.uct");

// RE
Path dir7 = fileSystem.getPath("/UCT/RE/2013/01/15");
Files.createDirectories(dir7);
createFile(dir7, "20130115_0030_RE2_FR0.uct");
createFile(dir7, "20130115_0130_RE2_FR0.uct");

// INTRADAY
Path dir8 = fileSystem.getPath("/UCT/IDCF/2013/01/15");
Files.createDirectories(dir8);
createFile(dir8, "20130115_0330_012_FR0.uct");
createFile(dir8, "20130115_0330_022_FR0.uct");
createFile(dir8, "20130115_0330_032_FR0.uct");

// daylight saving FO
Path dir9 = fileSystem.getPath("/UCT/FO/2016/10/30");
Files.createDirectories(dir9);
createFile(dir9, "20161030_0230_FO7_FR0.uct");
createFile(dir9, "20161030_B230_FO7_FR0.uct");
}

@After
Expand All @@ -169,7 +199,7 @@ public void testLoad() throws Exception {
// check that cim network is loaded instead of uct network
assertTrue(caseRepository.load(DateTime.parse("2013-01-14T00:15:00+01:00"), CaseType.SN, Country.FR).equals(Collections.singletonList(cimNetwork)));

// check that if cim is vorbidden for france, uct is loaded
// check that if cim is forbidden for france, uct is loaded
caseRepository.getConfig().getForbiddenFormatsByGeographicalCode().put(UcteGeographicalCode.FR, "CIM1");
assertTrue(caseRepository.load(DateTime.parse("2013-01-14T00:15:00+01:00"), CaseType.SN, Country.FR).equals(Collections.singletonList(uctNetwork)));

Expand All @@ -193,4 +223,71 @@ public void testDataAvailable() throws Exception {
assertTrue(caseRepository.dataAvailable(CaseType.SN, EnumSet.of(Country.FR), Interval.parse("2013-01-14T00:00:00+01:00/2013-01-14T01:00:00+01:00"))
.equals(Sets.newHashSet(DateTime.parse("2013-01-14T00:15:00+01:00"), DateTime.parse("2013-01-14T00:30:00+01:00"))));
}

@Test
public void testLoadD2() throws Exception {
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:30:00+01:00"), CaseType.D2, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:45:00+01:00"), CaseType.D2, Country.FR).isEmpty());
}

@Test
public void testLoadLT() throws Exception {
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:30:00+01:00"), CaseType.LT, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:45:00+01:00"), CaseType.LT, Country.FR).isEmpty());
}

@Test
public void testLoadRE() throws Exception {
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:30:00+01:00"), CaseType.RE, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T00:45:00+01:00"), CaseType.RE, Country.FR).isEmpty());
}

@Test
public void testLoadDayLightSaving() throws Exception {
List<Network> networksCEST=caseRepository.load(DateTime.parse("2016-10-30T02:30:00+02:00"), CaseType.FO, Country.FR);
assertTrue(networksCEST.size() == 1);
List<Network> networksCET=caseRepository.load(DateTime.parse("2016-10-30T02:30:00+01:00"), CaseType.FO, Country.FR);
assertTrue(networksCET.size() == 1);
}

@Test
public void testLoadIDCF() throws Exception {
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T03:30:00+01:00"), CaseType.IDCF01, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T03:30:00+01:00"), CaseType.IDCF02, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T03:30:00+01:00"), CaseType.IDCF03, Country.FR).size() == 1);
assertTrue(caseRepository.load(DateTime.parse("2013-01-15T03:30:00+01:00"), CaseType.IDCF04, Country.FR).isEmpty());
}

@Test
public void testIsDataAvailable2D() throws Exception {
assertTrue(caseRepository.isDataAvailable(DateTime.parse("2013-01-15T00:30:00+01:00"), CaseType.D2, Country.FR));
}

@Test
public void testDataAvailable2D() throws Exception {
assertTrue(caseRepository.dataAvailable(CaseType.D2, EnumSet.of(Country.FR), Interval.parse("2013-01-15T00:00:00+01:00/2013-01-15T01:30:00+01:00"))
.equals(Sets.newHashSet(DateTime.parse("2013-01-15T00:30:00+01:00"))));
}

@Test
public void testDataAvailableIntraday() throws Exception {
Set<DateTime> dset=caseRepository.dataAvailable(CaseType.IDCF01, EnumSet.of(Country.FR), Interval.parse("2013-01-15T00:00:00+01:00/2013-01-15T05:30:00+01:00"));
System.out.println(dset);
assertTrue(dset.equals(Sets.newHashSet(DateTime.parse("2013-01-15T03:30:00+01:00"))));
}

@Test
public void testDataAvailableDayLightSaving() throws Exception {

// double date CEST + CET
Set<DateTime> dset=caseRepository.dataAvailable(CaseType.FO, EnumSet.of(Country.FR), Interval.parse("2016-10-30T00:00:00+02:00/2016-10-30T03:30:00+01:00"));
System.out.println(dset);
assertTrue(dset.equals(Sets.newHashSet(DateTime.parse("2016-10-30T02:30:00+02:00"),DateTime.parse("2016-10-30T02:30:00+01:00"))));

//just the CET one
dset=caseRepository.dataAvailable(CaseType.FO, EnumSet.of(Country.FR), Interval.parse("2016-10-30T02:30:00+01:00/2016-10-30T03:30:00+01:00"));
System.out.println(dset);
assertTrue(dset.equals(Sets.newHashSet(DateTime.parse("2016-10-30T02:30:00+01:00"))));
}

}
8 changes: 4 additions & 4 deletions cim1-import/src/main/java/eu/itesla_project/iidm/import_/cim1/CIM1Converter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Multimap;
import eu.itesla_project.iidm.network.*;
import eu.itesla_project.ucte.util.UcteFileName;
import eu.itesla_project.ucte.util.EntsoeFileName;
import org.jgrapht.UndirectedGraph;
import org.jgrapht.alg.ConnectivityInspector;
import org.jgrapht.graph.Pseudograph;
Expand Down Expand Up @@ -1191,11 +1191,11 @@ Network convert() {
}
}

UcteFileName ucteFileName = UcteFileName.parse(fileName);
EntsoeFileName entsoeFileName = EntsoeFileName.parse(fileName);

Network network = NetworkFactory.create(fileName, FORMAT);
network.setCaseDate(ucteFileName.getDate());
network.setForecastDistance(ucteFileName.getForecastDistance());
network.setCaseDate(entsoeFileName.getDate());
network.setForecastDistance(entsoeFileName.getForecastDistance());

// Ends of transformers need to be in the same substation in the IIDM model, so check that a mapping is
// not needed
Expand Down
Loading