diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java index 1f55c1e136..5e65c0874c 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java @@ -42,6 +42,8 @@ public abstract class GtfsInput implements Closeable { public static final String invalidInputMessage = "At least 1 GTFS file is in a subfolder. All GTFS files must reside at the root level directly."; + public static final String USER_AGENT_PREFIX = "MobilityData GTFS-Validator"; + /** * Creates a specific GtfsInput to read data from the given path. * @@ -92,18 +94,6 @@ public static boolean hasSubfolderWithGtfsFile(Path path) throws IOException { return containsGtfsFileInSubfolder(zipInputStream); } - /** - * Check if an input zip file from an URL contains a subfolder with GTFS files - * - * @param url - * @return - * @throws IOException - */ - public static boolean hasSubfolderWithGtfsFile(URL url) throws IOException { - ZipInputStream zipInputStream = new ZipInputStream(new BufferedInputStream(url.openStream())); - return containsGtfsFileInSubfolder(zipInputStream); - } - /** * Common method used by two overloaded hasSubfolderWithGtfsFile methods * @@ -143,12 +133,13 @@ private static boolean containsGtfsFileInSubfolder(ZipInputStream zipInputStream * @param sourceUrl the fully qualified URL to download of the resource to download * @param targetPath the path to store the downloaded GTFS archive * @param noticeContainer + * @param validatorVersion * @return the {@code GtfsInput} created after download of the GTFS archive * @throws IOException if GTFS archive cannot be stored at the specified location * @throws URISyntaxException if URL is malformed */ public static GtfsInput createFromUrl( - URL sourceUrl, Path targetPath, NoticeContainer noticeContainer) + URL sourceUrl, Path targetPath, NoticeContainer noticeContainer, String validatorVersion) throws IOException, URISyntaxException { // getParent() may return null if there is no parent, so call toAbsolutePath() first. Path targetDirectory = targetPath.toAbsolutePath().getParent(); @@ -156,7 +147,7 @@ public static GtfsInput createFromUrl( Files.createDirectories(targetDirectory); } try (OutputStream outputStream = Files.newOutputStream(targetPath)) { - loadFromUrl(sourceUrl, outputStream); + loadFromUrl(sourceUrl, outputStream, validatorVersion); } return createFromPath(targetPath, noticeContainer); } @@ -171,14 +162,15 @@ public static GtfsInput createFromUrl( * @throws IOException if no file could not be found at the specified location * @throws URISyntaxException if URL is malformed */ - public static GtfsInput createFromUrlInMemory(URL sourceUrl, NoticeContainer noticeContainer) + public static GtfsInput createFromUrlInMemory( + URL sourceUrl, NoticeContainer noticeContainer, String validatorVersion) throws IOException, URISyntaxException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { - loadFromUrl(sourceUrl, outputStream); + loadFromUrl(sourceUrl, outputStream, validatorVersion); File zipFile = new File(sourceUrl.toString()); String fileName = zipFile.getName().replace(".zip", ""); - if (hasSubfolderWithGtfsFile(sourceUrl)) { - + if (containsGtfsFileInSubfolder( + new ZipInputStream(new ByteArrayInputStream(outputStream.toByteArray())))) { noticeContainer.addValidationNotice( new InvalidInputFilesInSubfolderNotice(invalidInputMessage)); } @@ -192,19 +184,35 @@ public static GtfsInput createFromUrlInMemory(URL sourceUrl, NoticeContainer not * * @param sourceUrl the fully qualified URL * @param outputStream the output stream + * @param validatorVersion * @throws IOException if no file could not be found at the specified location * @throws URISyntaxException if URL is malformed */ - private static void loadFromUrl(URL sourceUrl, OutputStream outputStream) + private static void loadFromUrl(URL sourceUrl, OutputStream outputStream, String validatorVersion) throws IOException, URISyntaxException { try (CloseableHttpClient httpClient = HttpClients.createDefault()) { HttpGet httpGet = new HttpGet(sourceUrl.toURI()); + httpGet.setHeader("User-Agent", getUserAgent(validatorVersion)); try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) { httpResponse.getEntity().writeTo(outputStream); } } } + /** + * @param validatorVersion version of the validator + * @return the user agent string in the format: "MobilityData GTFS-Validator/{validatorVersion} + * (Java {java version})" + */ + private static String getUserAgent(String validatorVersion) { + return USER_AGENT_PREFIX + + "/" + + (validatorVersion != null ? validatorVersion : "") + + " (Java " + + System.getProperty("java.version") + + ")"; + } + /** * Lists all files inside the GTFS dataset, even if they are not CSV and do not have .txt * extension. diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/input/GtfsInputTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/input/GtfsInputTest.java index 29539107d2..381471feb6 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/input/GtfsInputTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/input/GtfsInputTest.java @@ -77,17 +77,14 @@ public void zipInput() throws IOException { } } - @Test - public void urlInputHasNoSubfolderWithGtfsFile() throws IOException { - URL url = new URL(VALID_URL); - assertFalse(GtfsInput.hasSubfolderWithGtfsFile(url)); - } - @Test public void createFromUrl_valid_success() throws IOException, URISyntaxException { try (GtfsInput underTest = GtfsInput.createFromUrl( - new URL(VALID_URL), tmpDir.getRoot().toPath().resolve("storage"), noticeContainer)) { + new URL(VALID_URL), + tmpDir.getRoot().toPath().resolve("storage"), + noticeContainer, + "1.0.1")) { assertThat(underTest instanceof GtfsZipFileInput); } } @@ -100,13 +97,14 @@ public void createFromUrl_invalid_throwsException() { GtfsInput.createFromUrl( new URL(INVALID_URL), tmpDir.getRoot().toPath().resolve("storage"), - noticeContainer)); + noticeContainer, + "1.0.1")); } @Test public void createFromUrlInMemory_valid_success() throws IOException, URISyntaxException { try (GtfsInput underTest = - GtfsInput.createFromUrlInMemory(new URL(VALID_URL), noticeContainer)) { + GtfsInput.createFromUrlInMemory(new URL(VALID_URL), noticeContainer, "1.0.1")) { assertThat(underTest instanceof GtfsZipFileInput); } } @@ -115,6 +113,6 @@ public void createFromUrlInMemory_valid_success() throws IOException, URISyntaxE public void createFromUrlInMemory_invalid_throwsException() { assertThrows( IOException.class, - () -> GtfsInput.createFromUrlInMemory(new URL(INVALID_URL), noticeContainer)); + () -> GtfsInput.createFromUrlInMemory(new URL(INVALID_URL), noticeContainer, "1.0.1")); } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java index b06f52a4ee..0bb04fb5a6 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java @@ -102,7 +102,7 @@ public Status run(ValidationRunnerConfig config) { GtfsFeedContainer feedContainer; GtfsInput gtfsInput = null; try { - gtfsInput = createGtfsInput(config); + gtfsInput = createGtfsInput(config, versionInfo.currentVersion().get()); } catch (IOException e) { logger.atSevere().withCause(e).log("Cannot load GTFS feed"); noticeContainer.addSystemError(new IOError(e)); @@ -305,11 +305,12 @@ public static void exportReport( * Creates a {@code GtfsInput} * * @param config used to retrieve information needed to the creation of the {@code GtfsInput} + * @param validatorVersion version of the validator * @return the {@code GtfsInput} generated after * @throws IOException in case of error while loading a file * @throws URISyntaxException in case of error in the {@code URL} syntax */ - public static GtfsInput createGtfsInput(ValidationRunnerConfig config) + public static GtfsInput createGtfsInput(ValidationRunnerConfig config, String validatorVersion) throws IOException, URISyntaxException { URI source = config.gtfsSource(); if (source.getScheme().equals("file")) { @@ -317,12 +318,13 @@ public static GtfsInput createGtfsInput(ValidationRunnerConfig config) } if (config.storageDirectory().isEmpty()) { - return GtfsInput.createFromUrlInMemory(source.toURL(), noticeContainer); + return GtfsInput.createFromUrlInMemory(source.toURL(), noticeContainer, validatorVersion); } else { return GtfsInput.createFromUrl( source.toURL(), config.storageDirectory().get().resolve(GTFS_ZIP_FILENAME), - noticeContainer); + noticeContainer, + validatorVersion); } } } diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerTest.java index 09e3ee80b7..22730c2f4b 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunnerTest.java @@ -30,7 +30,8 @@ public void createGtfsInput_WindowsPath_valid() throws IOException, URISyntaxExc // We are testing path parsing here only. We expect a FileNotFoundException but NOT a // InvalidPathException. This should catch issues such as #1158. - assertThrows(FileNotFoundException.class, () -> ValidationRunner.createGtfsInput(config)); + assertThrows( + FileNotFoundException.class, () -> ValidationRunner.createGtfsInput(config, "1.1.0")); } @Test @@ -39,6 +40,7 @@ public void createGtfsInput_LinuxPath_valid() throws IOException, URISyntaxExcep // We are testing path parsing here only. We expect a FileNotFoundException but NOT a // InvalidPathException. This should catch issues such as #1158. - assertThrows(FileNotFoundException.class, () -> ValidationRunner.createGtfsInput(config)); + assertThrows( + FileNotFoundException.class, () -> ValidationRunner.createGtfsInput(config, "1.1.0")); } }