Skip to content

Commit

Permalink
Merge pull request #1246 from Isira-Seneviratne/BufferedReader-lines
Browse files Browse the repository at this point in the history 
Use BufferedReader#lines()
  • Loading branch information
@litetex
litetex authored Jan 27, 2025
2 parents d5ade94 + 222d869 commit 3a33cef
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 79 deletions.
87 changes: 23 additions & 64 deletions ...rg/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

Expand Down Expand Up @@ -110,7 +113,7 @@ public List<SubscriptionItem> fromZipInputStream(@Nonnull final InputStream cont

// Return it only if it has items (it exits early if it's the wrong file
// format), otherwise try the next file
if (csvItems.size() > 0) {
if (!csvItems.isEmpty()) {
return csvItems;
}
} catch (final ExtractionException e) {
Expand Down Expand Up @@ -138,69 +141,25 @@ public List<SubscriptionItem> fromCsvInputStream(@Nonnull final InputStream cont
// The first line is always a header
// Header names are different based on the locale
// Fortunately the data is always the same order no matter what locale

int currentLine = 0;
String line = "";

try (BufferedReader br = new BufferedReader(new InputStreamReader(contentInputStream))) {
final List<SubscriptionItem> subscriptionItems = new ArrayList<>();

// ignore header and skip first line
currentLine = 1;
line = br.readLine();

while ((line = br.readLine()) != null) {
currentLine++;

// Exit early if we've read the first few lines and we haven't added any items
// It's likely we're in the wrong file
if (currentLine > 5 && subscriptionItems.size() == 0) {
break;
}

// First comma
final int i1 = line.indexOf(",");
if (i1 == -1) {
continue;
}

// Second comma
final int i2 = line.indexOf(",", i1 + 1);
if (i2 == -1) {
continue;
}

// Third comma or line length
int i3 = line.indexOf(",", i2 + 1);
if (i3 == -1) {
i3 = line.length();
}

// Channel URL from second entry
final String channelUrl = line
.substring(i1 + 1, i2)
.replace("http://", "https://");
if (!channelUrl.startsWith(BASE_CHANNEL_URL)) {
continue;
}

// Channel title from third entry
final String channelTitle = line.substring(i2 + 1, i3);

final SubscriptionItem newItem
= new SubscriptionItem(service.getServiceId(), channelUrl, channelTitle);
subscriptionItems.add(newItem);
}

return subscriptionItems;
} catch (final IOException e) {
if (line == null) {
line = "<null>";
} else if (line.length() > 10) {
line = line.substring(0, 10) + "...";
}
throw new InvalidSourceException("Error reading CSV file on line = \"" + line
+ "\", line number = " + currentLine, e);
try (var reader = new BufferedReader(new InputStreamReader(contentInputStream))) {
return reader.lines()
.skip(1) // ignore header and skip first line
.map(line -> line.split(","))
.filter(values -> values.length >= 3)
.map(values -> {
// Channel URL from second entry
final String channelUrl = values[1].replace("http://", "https://");
return channelUrl.startsWith(BASE_CHANNEL_URL)
? new SubscriptionItem(
service.getServiceId(),
channelUrl,
values[2]) // Channel title from third entry
: null;
})
.filter(Objects::nonNull)
.collect(Collectors.toUnmodifiableList());
} catch (final UncheckedIOException | IOException e) {
throw new InvalidSourceException("Error reading CSV file", e);
}
}
}
30 changes: 15 additions & 15 deletions .../java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@
/**
* Test for {@link YoutubeSubscriptionExtractor}
*/
public class YoutubeSubscriptionExtractorTest {

class YoutubeSubscriptionExtractorTest {

private static YoutubeSubscriptionExtractor subscriptionExtractor;
private static LinkHandlerFactory urlHandler;
Expand All @@ -41,7 +40,7 @@ public static void setupClass() {
}

@Test
public void testFromInputStream() throws Exception {
void testFromInputStream() throws Exception {
final List<SubscriptionItem> subscriptionItems = subscriptionExtractor.fromInputStream(
new FileInputStream(resolveTestResource("youtube_takeout_import_test.json")));
assertEquals(7, subscriptionItems.size());
Expand All @@ -55,14 +54,14 @@ public void testFromInputStream() throws Exception {
}

@Test
public void testEmptySourceException() throws Exception {
void testEmptySourceException() throws Exception {
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)));
assertTrue(items.isEmpty());
}

@Test
public void testSubscriptionWithEmptyTitleInSource() throws Exception {
void testSubscriptionWithEmptyTitleInSource() throws Exception {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
Expand All @@ -74,7 +73,7 @@ public void testSubscriptionWithEmptyTitleInSource() throws Exception {
}

@Test
public void testSubscriptionWithInvalidUrlInSource() throws Exception {
void testSubscriptionWithInvalidUrlInSource() throws Exception {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
"{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
Expand All @@ -87,8 +86,8 @@ public void testSubscriptionWithInvalidUrlInSource() throws Exception {
}

@Test
public void testInvalidSourceException() {
List<String> invalidList = Arrays.asList(
void testInvalidSourceException() {
final List<String> invalidList = Arrays.asList(
"<xml><notvalid></notvalid></xml>",
"<opml><notvalid></notvalid></opml>",
"{\"a\":\"b\"}",
Expand All @@ -100,13 +99,14 @@ public void testInvalidSourceException() {
"\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28",
"gibberish");

for (String invalidContent : invalidList) {
for (final String invalidContent : invalidList) {
try {
byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
final byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
} catch (final Exception e) {
boolean correctType = e instanceof SubscriptionExtractor.InvalidSourceException;
final boolean correctType =
e instanceof SubscriptionExtractor.InvalidSourceException;
if (!correctType) {
e.printStackTrace();
}
Expand All @@ -117,7 +117,7 @@ public void testInvalidSourceException() {

private static void assertSubscriptionItems(final List<SubscriptionItem> subscriptionItems)
throws Exception {
assertTrue(subscriptionItems.size() > 0);
assertTrue(!subscriptionItems.isEmpty());

for (final SubscriptionItem item : subscriptionItems) {
assertNotNull(item.getName());
Expand All @@ -128,7 +128,7 @@ private static void assertSubscriptionItems(final List<SubscriptionItem> subscri
}

@Test
public void fromZipInputStream() throws Exception {
void fromZipInputStream() throws Exception {
final List<String> zipPaths = Arrays.asList(
"youtube_takeout_import_test_1.zip",
"youtube_takeout_import_test_2.zip"
Expand All @@ -144,13 +144,13 @@ public void fromZipInputStream() throws Exception {
}

@Test
public void fromCsvInputStream() throws Exception {
void fromCsvInputStream() throws Exception {
final List<String> csvPaths = Arrays.asList(
"youtube_takeout_import_test_1.csv",
"youtube_takeout_import_test_2.csv"
);

for (String path : csvPaths)
for (final String path : csvPaths)
{
final File file = resolveTestResource(path);
final FileInputStream fileInputStream = new FileInputStream(file);
Expand Down

0 comments on commit 3a33cef

Please sign in to comment.