diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java index 3faff9c3fa..2d1e4a0d7f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java @@ -21,8 +21,10 @@ * These tokens may have a role in triggering the sign in requirement. *

* - * @implNote This interface is expected to be thread-safe, - * as it may be accessed by multiple threads. + *

+ * Implementations of this interface are expected to be thread-safe, as they may be accessed by + * multiple threads. + *

*/ public interface PoTokenProvider { @@ -35,11 +37,77 @@ public interface PoTokenProvider { * must be added to adaptive/DASH streaming URLs with the {@code pot} parameter. *

* + *

+ * Note that YouTube desktop website generates two poTokens: + * - one for the player requests poTokens, using the videoId as the minter value; + * - one for the streaming URLs, using a visitor data for logged-out users. + *

+ * * @return a {@link PoTokenResult} specific to the WEB InnerTube client */ @Nullable - PoTokenResult getWebClientPoToken(); + PoTokenResult getWebClientPoToken(String videoId); + /** + * Get a {@link PoTokenResult} specific to the web embeds, a.k.a. the WEB_EMBEDDED_PLAYER + * InnerTube client. + * + *

+ * To be generated and valid, poTokens from this client must be generated using Google's + * BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They + * should be added to adaptive/DASH streaming URLs with the {@code pot} parameter and do not + * seem to be mandatory for now. + *

+ * + *

+ * As of writing, like the YouTube desktop website previously did, it generates only one + * poToken, sent in player requests and streaming URLs, using a visitor data for logged-out + * users. + *

+ * + * @return a {@link PoTokenResult} specific to the WEB_EMBEDDED_PLAYER InnerTube client + */ + @Nullable + PoTokenResult getWebEmbedClientPoToken(String videoId); + + /** + * Get a {@link PoTokenResult} specific to the Android app, a.k.a. the ANDROID InnerTube client. + * + *

+ * Implementation details are not known, the app uses DroidGuard, a native virtual machine + * ran by Google Play Services for which its code is updated pretty frequently. + *

+ * + *

+ * As of writing, DroidGuard seem to check for the Android app signature and package ID, as + * unrooted YouTube patched with reVanced doesn't work without spoofing another InnerTube + * client while the rooted version works without any client spoofing. + *

+ * + *

+ * There should be only poToken needed, for the player requests. + *

+ * + * @return a {@link PoTokenResult} specific to the ANDROID InnerTube client + */ + @Nullable + PoTokenResult getAndroidClientPoToken(String videoId); + + /** + * Get a {@link PoTokenResult} specific to the Android app, a.k.a. the ANDROID InnerTube client. + * + *

+ * Implementation details are not really known, the app seem to use something called + * iosGuard which should be something similar to Android's DroidGuard. It may rely on Apple's + * attestation APIs. + *

+ * + *

+ * There should be only poToken needed, for the player requests. + *

+ * + * @return a {@link PoTokenResult} specific to the IOS InnerTube client + */ @Nullable - PoTokenResult getAndroidClientPoToken(); + PoTokenResult getIosClientPoToken(String videoId); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java index af2520870e..aa21e74324 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java @@ -1,22 +1,40 @@ package org.schabi.newpipe.extractor.services.youtube; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.Objects; public final class PoTokenResult { /** - * The visitor data associated with a poToken. + * The visitor data associated with a {@code poToken}. */ + @Nonnull public final String visitorData; /** - * The poToken, a Protobuf object encoded as a base 64 string. + * The {@code poToken} of a player request, a Protobuf object encoded as a base 64 string. */ - public final String poToken; + @Nonnull + public final String playerRequestPoToken; - public PoTokenResult(@Nonnull final String visitorData, @Nonnull final String poToken) { + /** + * The {@code poToken} to be appended to streaming URLs, a Protobuf object encoded as a base + * 64 string. + * + *

+ * It may be required on some clients such as HTML5 ones and may also differ from the player + * request {@code poToken}. + *

+ */ + @Nullable + public final String streamingDataPoToken; + + public PoTokenResult(@Nonnull final String visitorData, + @Nonnull final String playerRequestPoToken, + @Nullable final String streamingDataPoToken) { this.visitorData = Objects.requireNonNull(visitorData); - this.poToken = Objects.requireNonNull(poToken); + this.playerRequestPoToken = Objects.requireNonNull(playerRequestPoToken); + this.streamingDataPoToken = streamingDataPoToken; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 1fd4499cdf..6b87425319 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -25,9 +25,7 @@ import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_OS_VERSION; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION; -import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION; @@ -1062,43 +1060,6 @@ public static JsonObject getJsonPostResponse(final String endpoint, + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization))); } - public static JsonObject getJsonAndroidPostResponse( - final String endpoint, - final byte[] body, - @Nonnull final Localization localization, - @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { - return getMobilePostResponse(endpoint, body, localization, - getAndroidUserAgent(localization), endPartOfUrlRequest); - } - - public static JsonObject getJsonIosPostResponse( - final String endpoint, - final byte[] body, - @Nonnull final Localization localization, - @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { - return getMobilePostResponse(endpoint, body, localization, getIosUserAgent(localization), - endPartOfUrlRequest); - } - - private static JsonObject getMobilePostResponse( - final String endpoint, - final byte[] body, - @Nonnull final Localization localization, - @Nonnull final String userAgent, - @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { - final var headers = Map.of("User-Agent", List.of(userAgent), - "X-Goog-Api-Format-Version", List.of("2")); - - final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?" - + DISABLE_PRETTY_PRINT_PARAMETER; - - return JsonUtils.toJsonObject(getValidJsonResponseBody( - getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest) - ? baseEndpointUrl - : baseEndpointUrl + endPartOfUrlRequest, - headers, body, localization))); - } - @Nonnull public static JsonBuilder prepareDesktopJsonBuilder( @Nonnull final Localization localization, @@ -1145,152 +1106,6 @@ public static JsonBuilder prepareDesktopJsonBuilder( // @formatter:on } - @Nonnull - public static JsonBuilder prepareAndroidMobileJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nullable final String visitorData) { - // @formatter:off - final JsonBuilder builder = JsonObject.builder() - .object("context") - .object("client") - .value("clientName", "ANDROID") - .value("clientVersion", ANDROID_CLIENT_VERSION) - .value("platform", "MOBILE") - .value("osName", "Android") - .value("osVersion", "14") - /* - A valid Android SDK version is required to be sure to get a valid player - response - If this parameter is not provided, the player response is replaced by an - error saying the message "The following content is not available on this - app. Watch this content on the latest version on YouTube" (it was - previously a 5-minute video with this message) - See https://github.com/TeamNewPipe/NewPipe/issues/8713 - The Android SDK version corresponding to the Android version used in - requests is sent - */ - .value("androidSdkVersion", 34) - .value("hl", localization.getLocalizationCode()) - .value("gl", contentCountry.getCountryCode()) - .value("utcOffsetMinutes", 0); - - if (visitorData != null) { - builder.value("visitorData", visitorData); - } - - builder.end() - .object("request") - .array("internalExperimentFlags") - .end() - .value("useSsl", true) - .end() - .object("user") - // TODO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) - .value("lockedSafetyMode", false) - .end() - .end(); - // @formatter:on - return builder; - } - - @Nonnull - public static JsonBuilder prepareIosMobileJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry) { - // @formatter:off - return JsonObject.builder() - .object("context") - .object("client") - .value("clientName", "IOS") - .value("clientVersion", IOS_CLIENT_VERSION) - .value("deviceMake", "Apple") - // Device model is required to get 60fps streams - .value("deviceModel", IOS_DEVICE_MODEL) - .value("platform", "MOBILE") - .value("osName", "iOS") - .value("osVersion", IOS_OS_VERSION) - .value("visitorData", randomVisitorData(contentCountry)) - .value("hl", localization.getLocalizationCode()) - .value("gl", contentCountry.getCountryCode()) - .value("utcOffsetMinutes", 0) - .end() - .object("request") - .array("internalExperimentFlags") - .end() - .value("useSsl", true) - .end() - .object("user") - // TODO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) - .value("lockedSafetyMode", false) - .end() - .end(); - // @formatter:on - } - - @Nonnull - public static JsonBuilder prepareTvHtml5EmbedJsonBuilder( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nonnull final String videoId) { - // @formatter:off - return JsonObject.builder() - .object("context") - .object("client") - .value("clientName", "TVHTML5_SIMPLY_EMBEDDED_PLAYER") - .value("clientVersion", TVHTML5_CLIENT_VERSION) - .value("clientScreen", "EMBED") - .value("platform", "TV") - .value("hl", localization.getLocalizationCode()) - .value("gl", contentCountry.getCountryCode()) - .value("utcOffsetMinutes", 0) - .end() - .object("thirdParty") - .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId) - .end() - .object("request") - .array("internalExperimentFlags") - .end() - .value("useSsl", true) - .end() - .object("user") - // TODO: provide a way to enable restricted mode with: - // .value("enableSafetyMode", boolean) - .value("lockedSafetyMode", false) - .end() - .end(); - // @formatter:on - } - - @Nonnull - public static byte[] createTvHtml5EmbedPlayerBody( - @Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nonnull final String videoId, - @Nonnull final Integer sts, - @Nonnull final String contentPlaybackNonce) { - // @formatter:off - return JsonWriter.string( - prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId) - .object("playbackContext") - .object("contentPlaybackContext") - // Signature timestamp from the JavaScript base player is needed to get - // working obfuscated URLs - .value("signatureTimestamp", sts) - .value("referer", "https://www.youtube.com/watch?v=" + videoId) - .end() - .end() - .value(CPN, contentPlaybackNonce) - .value(VIDEO_ID, videoId) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) - .getBytes(StandardCharsets.UTF_8); - // @formatter:on - } - /** * Get the user-agent string used as the user-agent for InnerTube requests with the Android * client. @@ -1371,7 +1186,7 @@ public static Map> getClientInfoHeaders() * * @param url The URL to be set as the origin and referrer. */ - private static Map> getOriginReferrerHeaders(@Nonnull final String url) { + static Map> getOriginReferrerHeaders(@Nonnull final String url) { final var urlList = List.of(url); return Map.of("Origin", urlList, "Referer", urlList); } @@ -1383,8 +1198,8 @@ private static Map> getOriginReferrerHeaders(@Nonnull final * @param name The X-YouTube-Client-Name value. * @param version X-YouTube-Client-Version value. */ - private static Map> getClientHeaders(@Nonnull final String name, - @Nonnull final String version) { + static Map> getClientHeaders(@Nonnull final String name, + @Nonnull final String version) { return Map.of("X-YouTube-Client-Name", List.of(name), "X-YouTube-Client-Version", List.of(version)); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java index 98d17591ce..5b4276e628 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube; +import com.grack.nanojson.JsonBuilder; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -8,28 +9,53 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.EMBED_CLIENT_SCREEN; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_OS_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.MOBILE_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_PLATFORM; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WATCH_CLIENT_SCREEN; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_NAME; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_VERSION; +import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_GAPIS_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientHeaders; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getOriginReferrerHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public final class YoutubeStreamHelper { - private static final String STREAMING_DATA = "streamingData"; private static final String PLAYER = "player"; private static final String SERVICE_INTEGRITY_DIMENSIONS = "serviceIntegrityDimensions"; private static final String PO_TOKEN = "poToken"; @@ -42,14 +68,27 @@ public static JsonObject getWebMetadataPlayerResponse( @Nonnull final Localization localization, @Nonnull final ContentCountry contentCountry, @Nonnull final String videoId) throws IOException, ExtractionException { - final byte[] body = JsonWriter.string( - prepareDesktopJsonBuilder(localization, contentCountry) - .value(VIDEO_ID, videoId) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + WEB_CLIENT_NAME, + getClientVersion(), + WATCH_CLIENT_SCREEN, + DESKTOP_CLIENT_PLATFORM, + "", // TODO + null, + null, + null, + null, + null, + -1); + + addVideoIdCpnAndOkChecks(builder, videoId, null); + + final byte[] body = JsonWriter.string(builder.done()) .getBytes(StandardCharsets.UTF_8); - final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER + + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER + "&$fields=microformat,playabilityStatus,storyboards,videoDetails"; return JsonUtils.toJsonObject(getValidJsonResponseBody( @@ -57,60 +96,166 @@ public static JsonObject getWebMetadataPlayerResponse( url, getYouTubeHeaders(), body, localization))); } + @Nonnull + public static JsonObject getTvHtml5PlayerResponse( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nonnull final String cpn) throws IOException, ExtractionException { + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + TVHTML5_CLIENT_NAME, + TVHTML5_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + TVHTML5_CLIENT_PLATFORM, + YoutubeParsingHelper.randomVisitorData(contentCountry), + null, + null, + null, + null, + null, + -1); + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + final byte[] body = JsonWriter.string(builder.done()) + .getBytes(StandardCharsets.UTF_8); + + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; + + final Map> headers = new HashMap<>( + getClientHeaders(TVHTML5_CLIENT_ID, TVHTML5_CLIENT_VERSION)); + headers.putAll(getOriginReferrerHeaders("https://www.youtube.com")); + headers.put("User-Agent", List.of(TVHTML5_USER_AGENT)); + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson( + url, headers, body, localization))); + } + @Nonnull public static JsonObject getWebFullPlayerResponse( @Nonnull final Localization localization, @Nonnull final ContentCountry contentCountry, @Nonnull final String videoId, - @Nonnull final PoTokenResult webPoTokenResult) throws IOException, ExtractionException { - final byte[] body = JsonWriter.string( - prepareDesktopJsonBuilder( - localization, - contentCountry, - webPoTokenResult.visitorData - ) - .value(VIDEO_ID, videoId) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .object(SERVICE_INTEGRITY_DIMENSIONS) - .value(PO_TOKEN, webPoTokenResult.poToken) - .end() - .done()) + @Nonnull final String cpn, + @Nonnull final PoTokenResult webPoTokenResult, + final int signatureTimestamp) throws IOException, ExtractionException { + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + WEB_CLIENT_NAME, + getClientVersion(), + WATCH_CLIENT_SCREEN, + DESKTOP_CLIENT_PLATFORM, + webPoTokenResult.visitorData, + null, + null, + null, + null, + null, + -1); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + addPlaybackContext( + builder, + "https://www.youtube.com/watch?v=" + videoId, + signatureTimestamp); + + addPoToken(builder, webPoTokenResult.playerRequestPoToken); + + final byte[] body = JsonWriter.string(builder.done()) .getBytes(StandardCharsets.UTF_8); - final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER; + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; return JsonUtils.toJsonObject(getValidJsonResponseBody( getDownloader().postWithContentTypeJson( url, getYouTubeHeaders(), body, localization))); } + @Nonnull + public static JsonObject getWebEmbeddedPlayerResponse( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nonnull final String cpn, + @Nullable final PoTokenResult webEmbeddedPoTokenResult, + final int signatureTimestamp) throws IOException, ExtractionException { + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + WEB_EMBEDDED_CLIENT_NAME, + WEB_REMIX_HARDCODED_CLIENT_VERSION, + EMBED_CLIENT_SCREEN, + DESKTOP_CLIENT_PLATFORM, + webEmbeddedPoTokenResult == null + ? YoutubeParsingHelper.randomVisitorData(contentCountry) + : webEmbeddedPoTokenResult.visitorData, + null, + null, + null, + null, + null, + -1); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + addPlaybackContext( + builder, + "https://www.youtube.com/watch?v=" + videoId, + signatureTimestamp); + + if (webEmbeddedPoTokenResult != null) { + addPoToken(builder, webEmbeddedPoTokenResult.playerRequestPoToken); + } + + final byte[] body = JsonWriter.string(builder.end().done()) + .getBytes(StandardCharsets.UTF_8); + final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER; + + final Map> headers = new HashMap<>( + getClientHeaders(WEB_EMBEDDED_CLIENT_ID, WEB_EMBEDDED_CLIENT_VERSION)); + headers.putAll(getOriginReferrerHeaders("https://www.youtube.com")); + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson( + url, headers, body, localization))); + } + public static JsonObject getAndroidPlayerResponse( @Nonnull final ContentCountry contentCountry, @Nonnull final Localization localization, @Nonnull final String videoId, - @Nonnull final String androidCpn, - @Nonnull final PoTokenResult androidPoTokenResult - ) + @Nonnull final String cpn, + @Nonnull final PoTokenResult androidPoTokenResult) throws IOException, ExtractionException { - final byte[] mobileBody = JsonWriter.string( - prepareAndroidMobileJsonBuilder( - localization, - contentCountry, - androidPoTokenResult.visitorData - ) - .value(VIDEO_ID, videoId) - .value(CPN, androidCpn) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .object(SERVICE_INTEGRITY_DIMENSIONS) - .value(PO_TOKEN, androidPoTokenResult.poToken) - .end() - .done()) + + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + ANDROID_CLIENT_NAME, + ANDROID_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + MOBILE_CLIENT_PLATFORM, + androidPoTokenResult.visitorData, + null, + null, + "Android", + "15", + null, + 35); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + addPoToken(builder, androidPoTokenResult.playerRequestPoToken); + + final byte[] body = JsonWriter.string(builder.end().done()) .getBytes(StandardCharsets.UTF_8); return getJsonAndroidPostResponse( - "player", - mobileBody, + PLAYER, + body, localization, "&t=" + generateTParameter() + "&id=" + videoId); } @@ -119,20 +264,30 @@ public static JsonObject getAndroidReelPlayerResponse( @Nonnull final ContentCountry contentCountry, @Nonnull final Localization localization, @Nonnull final String videoId, - @Nonnull final String androidCpn - ) - throws IOException, ExtractionException { - final byte[] mobileBody = JsonWriter.string( - prepareAndroidMobileJsonBuilder(localization, contentCountry, null) - .object("playerRequest") - .value(VIDEO_ID, videoId) - .end() - .value("disablePlayerResponse", false) - .value(VIDEO_ID, videoId) - .value(CPN, androidCpn) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) + @Nonnull final String cpn) throws IOException, ExtractionException { + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + ANDROID_CLIENT_NAME, + ANDROID_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + MOBILE_CLIENT_PLATFORM, + "", // TODO + null, + null, + "Android", + "15", + null, + 35); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + + builder.object("playerRequest") + .value(VIDEO_ID, videoId) + .end() + .value("disablePlayerResponse", false); + + final byte[] mobileBody = JsonWriter.string(builder.done()) .getBytes(StandardCharsets.UTF_8); final JsonObject androidPlayerResponse = getJsonAndroidPostResponse( @@ -147,19 +302,169 @@ public static JsonObject getAndroidReelPlayerResponse( public static JsonObject getIosPlayerResponse(@Nonnull final ContentCountry contentCountry, @Nonnull final Localization localization, @Nonnull final String videoId, - @Nonnull final String iosCpn) + @Nonnull final String cpn, + @Nullable final PoTokenResult iosPoTokenResult) throws IOException, ExtractionException { - final byte[] mobileBody = JsonWriter.string( - prepareIosMobileJsonBuilder(localization, contentCountry) - .value(VIDEO_ID, videoId) - .value(CPN, iosCpn) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) + final boolean noPoTokenResult = iosPoTokenResult == null; + final JsonBuilder builder = prepareJsonBuilder( + localization, + contentCountry, + IOS_CLIENT_NAME, + IOS_CLIENT_VERSION, + WATCH_CLIENT_SCREEN, + MOBILE_CLIENT_PLATFORM, + noPoTokenResult ? "" : iosPoTokenResult.visitorData, // TODO + "Apple", + IOS_DEVICE_MODEL, + "iOS", + IOS_OS_VERSION, + null, + -1); + + addVideoIdCpnAndOkChecks(builder, videoId, cpn); + if (!noPoTokenResult) { + addPoToken(builder, iosPoTokenResult.playerRequestPoToken); + } + + final byte[] mobileBody = JsonWriter.string(builder.done()) .getBytes(StandardCharsets.UTF_8); - return getJsonIosPostResponse(PLAYER, + return getJsonIosPostResponse( mobileBody, localization, "&t=" + generateTParameter() - + "&id=" + videoId); + + "&id=" + videoId + "&fields=streamingData.hlsManifestUrl"); + } + + public static JsonObject getJsonAndroidPostResponse(final String endpoint, + final byte[] body, + @Nonnull final Localization localization, + @Nullable final String endPartOfUrlRequest) + throws IOException, ExtractionException { + return getMobilePostResponse(endpoint, body, localization, + getAndroidUserAgent(localization), endPartOfUrlRequest); + } + + private static JsonObject getJsonIosPostResponse(final byte[] body, + @Nonnull final Localization localization, + @Nullable final String endPartOfUrlRequest) + throws IOException, ExtractionException { + return getMobilePostResponse(YoutubeStreamHelper.PLAYER, body, localization, + getIosUserAgent(localization), + endPartOfUrlRequest); + } + + @Nonnull + private static JsonBuilder prepareJsonBuilder( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String clientName, + @Nonnull final String clientVersion, + @Nonnull final String clientScreen, + @Nonnull final String platform, + @Nonnull final String visitorData, + @Nullable final String deviceMake, + @Nullable final String deviceModel, + @Nullable final String osName, + @Nullable final String osVersion, + @Nullable final String embedUrl, + final int androidSdkVersion) { + final JsonBuilder builder = JsonObject.builder() + .object("context") + .object("client") + .value("clientName", clientName) + .value("clientVersion", clientVersion) + .value("clientScreen", clientScreen) + .value("platform", platform) + .value("visitorData", visitorData); + + if (deviceMake != null) { + builder.value("deviceMake", deviceMake); + } + if (deviceModel != null) { + builder.value("deviceModel", deviceModel); + } + if (osName != null) { + builder.value("osName", osName); + } + if (osVersion != null) { + builder.value("osVersion", osVersion); + } + if (androidSdkVersion > 0) { + builder.value("androidSdkVersion", androidSdkVersion); + } + + builder.value("hl", localization.getLocalizationCode()) + .value("gl", contentCountry.getCountryCode()) + .value("utcOffsetMinutes", 0) + .end(); + + if (embedUrl != null) { + builder.object("thirdParty") + .value("embedUrl", embedUrl) + .end(); + } + + builder.object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) + .end() + .object("user") + // TODO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) + .end() + .end(); + + return builder; + } + + private static JsonObject getMobilePostResponse(final String endpoint, + final byte[] body, + @Nonnull final Localization localization, + @Nonnull final String userAgent, + @Nullable final String endPartOfUrlRequest) + throws IOException, ExtractionException { + final var headers = Map.of("User-Agent", List.of(userAgent), + "X-Goog-Api-Format-Version", List.of("2")); + + final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?" + + DISABLE_PRETTY_PRINT_PARAMETER; + + return JsonUtils.toJsonObject(getValidJsonResponseBody( + getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest) + ? baseEndpointUrl + : baseEndpointUrl + endPartOfUrlRequest, + headers, body, localization))); + } + + private static void addVideoIdCpnAndOkChecks(@Nonnull final JsonBuilder builder, + @Nonnull final String videoId, + @Nullable final String cpn) { + builder.value(VIDEO_ID, videoId); + + if (cpn != null) { + builder.value(CPN, cpn); + } + + builder.value(CONTENT_CHECK_OK, true) + .value(RACY_CHECK_OK, true); + } + + private static void addPlaybackContext(@Nonnull final JsonBuilder builder, + @Nonnull final String referer, + final int signatureTimestamp) { + builder.object("playbackContext") + .object("contentPlaybackContext") + .value("signatureTimestamp", signatureTimestamp) + .value("referer", referer) + .end() + .end(); + } + + private static void addPoToken(@Nonnull final JsonBuilder builder, + @Nonnull final String poToken) { + builder.object(SERVICE_INTEGRITY_DIMENSIONS) + .value(PO_TOKEN, poToken) + .end(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 68705c7a22..9a9f769ab2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -101,8 +101,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { + public static final String PLAYER_CAPTIONS_TRACKLIST_RENDERER = + "playerCaptionsTracklistRenderer"; + public static final String CAPTIONS = "captions"; @Nullable private static PoTokenProvider poTokenProvider; + private static boolean forceFetchIosClient; private JsonObject playerResponse; private JsonObject nextResponse; @@ -127,9 +131,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { // three different strings are used. private String iosCpn; private String androidCpn; - private String tvHtml5SimplyEmbedCpn; + private String html5Cpn; - private static boolean forceFetchIosClient; + @Nullable + private String iosStreamingUrlsPoToken; + @Nullable + private String androidStreamingUrlsPoToken; + @Nullable + private String html5StreamingUrlsPoToken; public YoutubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) { super(service, linkHandler); @@ -323,7 +332,7 @@ public long getLength() throws ParsingException { return Long.parseLong(duration); } catch (final Exception e) { return getDurationFromFirstAdaptiveFormat(Arrays.asList( - iosStreamingData, androidStreamingData, html5StreamingData)); + androidStreamingData, html5StreamingData)); } } @@ -581,11 +590,13 @@ public long getUploaderSubscriberCount() throws ParsingException { public String getDashMpdUrl() throws ParsingException { assertPageFetched(); - // There is no DASH manifest available in the iOS clients and the DASH manifest of the - // Android client doesn't contain all available streams (mainly the WEBM ones) + // There is no DASH manifest available with the iOS clients return getManifestUrl( "dash", - Arrays.asList(androidStreamingData, html5StreamingData)); + Arrays.asList( + new Pair<>(androidStreamingData, androidStreamingUrlsPoToken), + new Pair<>(html5StreamingData, html5StreamingUrlsPoToken)), + "mpd_version=7"); } @Nonnull @@ -594,25 +605,42 @@ public String getHlsUrl() throws ParsingException { assertPageFetched(); // Return HLS manifest of the iOS client first because on livestreams, the HLS manifest - // returned has separated audio and video streams + // returned has separated audio and video streams and poTokens requirement do not seem to + // impact HLS formats // Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response + // as we don't use a Safari macOS user agent return getManifestUrl( "hls", Arrays.asList( - iosStreamingData, androidStreamingData, html5StreamingData)); + new Pair<>(iosStreamingData, iosStreamingUrlsPoToken), + new Pair<>(androidStreamingData, androidStreamingUrlsPoToken), + new Pair<>(html5StreamingData, html5StreamingUrlsPoToken)), + ""); } @Nonnull - private static String getManifestUrl(@Nonnull final String manifestType, - @Nonnull final List streamingDataObjects) { + private static String getManifestUrl( + @Nonnull final String manifestType, + @Nonnull final List> streamingDataObjects, + @Nonnull final String partToAppendToManifestUrlEnd) { final String manifestKey = manifestType + "ManifestUrl"; - return streamingDataObjects.stream() - .filter(Objects::nonNull) - .map(streamingDataObject -> streamingDataObject.getString(manifestKey)) - .filter(Objects::nonNull) - .findFirst() - .orElse(""); + for (final Pair streamingDataObj : streamingDataObjects) { + final String manifestUrl = streamingDataObj.getFirst().getString(manifestKey); + if (isNullOrEmpty(manifestUrl)) { + continue; + } + + // If poToken is not null, add it to manifest URL + if (streamingDataObj.getSecond() == null) { + return manifestUrl + "?" + partToAppendToManifestUrlEnd; + } else { + return manifestUrl + "?pot=" + streamingDataObj.getSecond() + "&=" + + partToAppendToManifestUrlEnd; + } + } + + return ""; } @Override @@ -784,9 +812,28 @@ public void onFetchPage(@Nonnull final Downloader downloader) final boolean noPoTokenProviderSet = providerInstance == null; final PoTokenResult webPoTokenResult = noPoTokenProviderSet ? null - : providerInstance.getWebClientPoToken(); + : providerInstance.getWebClientPoToken(videoId); + + fetchHtml5Client(localization, contentCountry, videoId, webPoTokenResult); + + // Use the player response from the player endpoint of the desktop internal API because + // there can be restrictions on videos in the embedded player. + // E.g. if a video is age-restricted, the embedded player's playabilityStatus says that + // the video cannot be played outside of YouTube, but does not show the original message. + final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus"); + + final boolean isAgeRestricted = "login_required".equalsIgnoreCase( + playabilityStatus.getString("status")) + && playabilityStatus.getString("reason", "") + .contains("age"); + + if (isAgeRestricted) { + final PoTokenResult webEmbedPoTokenResult = noPoTokenProviderSet ? null + : providerInstance.getWebEmbedClientPoToken(videoId); + + fetchHtml5EmbedClient(localization, contentCountry, videoId, webEmbedPoTokenResult); + } - fetchWebClient(localization, contentCountry, videoId, webPoTokenResult); setStreamType(); // The microformat JSON object of the content is only returned on the WEB client, @@ -794,32 +841,17 @@ public void onFetchPage(@Nonnull final Downloader downloader) playerMicroFormatRenderer = playerResponse.getObject("microformat") .getObject("playerMicroformatRenderer"); - checkPlayabilityStatus(playerResponse, playerResponse.getObject("playabilityStatus")); - - if (forceFetchIosClient || webPoTokenResult == null) { - iosCpn = generateContentPlaybackNonce(); - final JsonObject iosPlayerResponse = YoutubeStreamHelper.getIosPlayerResponse( - contentCountry, localization, videoId, iosCpn); - - if (isPlayerResponseNotValid(iosPlayerResponse, videoId)) { - throw new ExtractionException("IOS player response is not valid"); - } - - final JsonObject iosStreamingDataLocal = iosPlayerResponse.getObject(STREAMING_DATA); - if (!isNullOrEmpty(iosStreamingDataLocal)) { - this.iosStreamingData = iosStreamingDataLocal; - if (!forceFetchIosClient) { - playerCaptionsTracklistRenderer = iosPlayerResponse.getObject("captions") - .getObject("playerCaptionsTracklistRenderer"); - } - } - } - final PoTokenResult androidPoTokenResult = noPoTokenProviderSet ? null - : providerInstance.getAndroidClientPoToken(); + : providerInstance.getAndroidClientPoToken(videoId); fetchAndroidClient(localization, contentCountry, videoId, androidPoTokenResult); + if (forceFetchIosClient) { + final PoTokenResult iosPoTokenResult = noPoTokenProviderSet ? null + : providerInstance.getIosClientPoToken(videoId); + fetchIosClient(localization, contentCountry, videoId, iosPoTokenResult); + } + final byte[] nextBody = JsonWriter.string( prepareDesktopJsonBuilder(localization, contentCountry) .value(VIDEO_ID, videoId) @@ -830,31 +862,24 @@ public void onFetchPage(@Nonnull final Downloader downloader) nextResponse = getJsonPostResponse(NEXT, nextBody, localization); } - private static void checkPlayabilityStatus(final JsonObject youtubePlayerResponse, - @Nonnull final JsonObject playabilityStatus) + private static void checkPlayabilityStatus(@Nonnull final JsonObject playabilityStatus) throws ParsingException { - String status = playabilityStatus.getString("status"); + final String status = playabilityStatus.getString("status"); if (status == null || status.equalsIgnoreCase("ok")) { return; } - // If status exist, and is not "OK", throw the specific exception based on error message - // or a ContentNotAvailableException with the reason text if it's an unknown reason. - final JsonObject newPlayabilityStatus = - youtubePlayerResponse.getObject("playabilityStatus"); - status = newPlayabilityStatus.getString("status"); - final String reason = newPlayabilityStatus.getString("reason"); + final String reason = playabilityStatus.getString("reason"); if (status.equalsIgnoreCase("login_required")) { if (reason == null) { - final String message = newPlayabilityStatus.getArray("messages").getString(0); + final String message = playabilityStatus.getArray("messages").getString(0); if (message != null && message.contains("private")) { - throw new PrivateContentException("This video is private."); + throw new PrivateContentException("This video is private"); } } else if (reason.contains("age")) { throw new AgeRestrictedContentException( - "Age-restricted videos cannot be watched anonymously" - ); + "This age-restricted video cannot be watched anonymously"); } } @@ -872,7 +897,7 @@ private static void checkPlayabilityStatus(final JsonObject youtubePlayerRespons } if (reason.contains("unavailable")) { - final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus + final String detailedErrorMessage = getTextFromObject(playabilityStatus .getObject("errorScreen") .getObject("playerErrorMessageRenderer") .getObject("subreason")); @@ -889,40 +914,97 @@ private static void checkPlayabilityStatus(final JsonObject youtubePlayerRespons throw new ContentNotAvailableException("Got error: \"" + reason + "\""); } - private void fetchWebClient(@Nonnull final Localization localization, - @Nonnull final ContentCountry contentCountry, - @Nonnull final String videoId, - @Nullable final PoTokenResult webPoTokenResult - ) throws IOException, ExtractionException { + private void fetchHtml5Client(@Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nullable final PoTokenResult webPoTokenResult) + throws IOException, ExtractionException { + html5Cpn = generateContentPlaybackNonce(); + final JsonObject webPlayerResponse; if (webPoTokenResult == null) { webPlayerResponse = YoutubeStreamHelper.getWebMetadataPlayerResponse( localization, contentCountry, videoId); + + throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId); + + // Save the webPlayerResponse into playerResponse in the case the video cannot be + // played, so some metadata can be retrieved + playerResponse = webPlayerResponse; + + final JsonObject tvHtml5PlayerResponse = YoutubeStreamHelper.getTvHtml5PlayerResponse( + localization, contentCountry, videoId, html5Cpn); + + if (isPlayerResponseNotValid(tvHtml5PlayerResponse, videoId)) { + throw new ExtractionException("TVHTML5 player response is not valid"); + } + + html5StreamingData = tvHtml5PlayerResponse.getObject(STREAMING_DATA); + playerCaptionsTracklistRenderer = tvHtml5PlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); } else { webPlayerResponse = YoutubeStreamHelper.getWebFullPlayerResponse( - localization, contentCountry, videoId, webPoTokenResult); - final JsonObject webStreamingData = webPlayerResponse.getObject(STREAMING_DATA); - if (!isNullOrEmpty(webStreamingData)) { - html5StreamingData = webStreamingData; - playerCaptionsTracklistRenderer = webPlayerResponse.getObject("captions") - .getObject("playerCaptionsTracklistRenderer"); - } + localization, contentCountry, videoId, html5Cpn, webPoTokenResult, + YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId)); + + // Save the webPlayerResponse into playerResponse in the case the video cannot be + // played, so some metadata can be retrieved + playerResponse = webPlayerResponse; + + throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId); + + html5StreamingData = webPlayerResponse.getObject(STREAMING_DATA); + playerCaptionsTracklistRenderer = webPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + html5StreamingUrlsPoToken = webPoTokenResult.streamingDataPoToken; } + } + private static void throwExceptionIfPlayerResponseNotValid( + @Nonnull final JsonObject webPlayerResponse, + @Nonnull final String videoId) throws ExtractionException { if (isPlayerResponseNotValid(webPlayerResponse, videoId)) { - // Check the playability status, as private and deleted videos and invalid video IDs do - // not return the ID provided in the player response - // When the requested video is playable and a different video ID is returned, it has - // the OK playability status, meaning the ExtractionException after this check will be - // thrown - checkPlayabilityStatus( - webPlayerResponse, webPlayerResponse.getObject("playabilityStatus")); - throw new ExtractionException("Initial WEB player response is not valid"); + // Check the playability status, as private and deleted videos and invalid video + // IDs do not return the ID provided in the player response + // When the requested video is playable and a different video ID is returned, it + // has the OK playability status, meaning the ExtractionException after this check + // will be thrown + checkPlayabilityStatus(webPlayerResponse.getObject("playabilityStatus")); + throw new ExtractionException("WEB player response is not valid"); + } + } + + private void fetchHtml5EmbedClient(@Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nullable final PoTokenResult webEmbedPoTokenResult) + throws IOException, ExtractionException { + html5Cpn = generateContentPlaybackNonce(); + + final JsonObject webEmbeddedPlayerResponse = + YoutubeStreamHelper.getWebEmbeddedPlayerResponse(localization, contentCountry, + videoId, html5Cpn, webEmbedPoTokenResult, + YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId)); + + // Save the webEmbeddedPlayerResponse into playerResponse in the case the video cannot be + // played, so some metadata can be retrieved + playerResponse = webEmbeddedPlayerResponse; + + if (webEmbedPoTokenResult != null) { + html5StreamingUrlsPoToken = webEmbedPoTokenResult.streamingDataPoToken; + } + + // Check if the playability status in the player response, if the age-restriction could not + // be bypassed, an exception will be thrown + checkPlayabilityStatus(webEmbeddedPlayerResponse.getObject("playabilityStatus")); + if (isPlayerResponseNotValid(webEmbeddedPlayerResponse, videoId)) { + throw new ExtractionException("WEB_EMBEDDED_PLAYER player response is not valid"); } - // Save the webPlayerResponse into playerResponse in the case the video cannot be played, - // so some metadata can be retrieved - playerResponse = webPlayerResponse; + if (isNullOrEmpty(playerCaptionsTracklistRenderer)) { + playerCaptionsTracklistRenderer = webEmbeddedPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + } } private void fetchAndroidClient(@Nonnull final Localization localization, @@ -930,9 +1012,9 @@ private void fetchAndroidClient(@Nonnull final Localization localization, @Nonnull final String videoId, @Nullable final PoTokenResult androidPoTokenResult) { try { - final JsonObject androidPlayerResponse; androidCpn = generateContentPlaybackNonce(); + final JsonObject androidPlayerResponse; if (androidPoTokenResult == null) { androidPlayerResponse = YoutubeStreamHelper.getAndroidReelPlayerResponse( contentCountry, localization, videoId, androidCpn); @@ -943,15 +1025,12 @@ private void fetchAndroidClient(@Nonnull final Localization localization, } if (!isPlayerResponseNotValid(androidPlayerResponse, videoId)) { - final JsonObject androidStreamingDataLocal = - androidPlayerResponse.getObject(STREAMING_DATA); - if (!isNullOrEmpty(androidStreamingDataLocal)) { - this.androidStreamingData = androidStreamingDataLocal; - if (isNullOrEmpty(playerCaptionsTracklistRenderer)) { - playerCaptionsTracklistRenderer = - androidPlayerResponse.getObject("captions") - .getObject("playerCaptionsTracklistRenderer"); - } + androidStreamingData = androidPlayerResponse.getObject(STREAMING_DATA); + + if (isNullOrEmpty(playerCaptionsTracklistRenderer)) { + playerCaptionsTracklistRenderer = + androidPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); } } } catch (final Exception ignored) { @@ -960,6 +1039,30 @@ private void fetchAndroidClient(@Nonnull final Localization localization, } } + private void fetchIosClient(@Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nullable final PoTokenResult iosPoTokenResult) { + try { + iosCpn = generateContentPlaybackNonce(); + + final JsonObject iosPlayerResponse = YoutubeStreamHelper.getIosPlayerResponse( + contentCountry, localization, videoId, iosCpn, iosPoTokenResult); + + if (!isPlayerResponseNotValid(iosPlayerResponse, videoId)) { + iosStreamingData = iosPlayerResponse.getObject(STREAMING_DATA); + + if (isNullOrEmpty(playerCaptionsTracklistRenderer)) { + playerCaptionsTracklistRenderer = iosPlayerResponse.getObject(CAPTIONS) + .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER); + } + } + } catch (final Exception ignored) { + // Ignore exceptions related to IOS client fetch or parsing, as it is not + // compulsory to play contents + } + } + /** * Checks whether a player response is invalid. * @@ -1049,22 +1152,28 @@ private List getItags( java.util.stream.Stream.of( /* - Use the iosStreamingData object first because there is no n param and no - signatureCiphers in streaming URLs of the iOS client + Use the html5StreamingData object first because YouTube should have less + control on HTML5 clients, especially for poTokens - The androidStreamingData is used as second way as it isn't used on livestreams, - it doesn't return all available streams, and the Android client extraction is - more likely to break + The androidStreamingData is used as second way as the Android client extraction + is more likely to break - As age-restricted videos are not common, use tvHtml5SimplyEmbedStreamingData - last, which will be the only one not empty for age-restricted content + As iOS streaming data is affected by poTokens and not passing them should lead + to 403 responses, it should be used in the last resort */ - new Pair<>(iosStreamingData, iosCpn), - new Pair<>(androidStreamingData, androidCpn), - new Pair<>(html5StreamingData, tvHtml5SimplyEmbedCpn) - ) - .flatMap(pair -> getStreamsFromStreamingDataKey(videoId, pair.getFirst(), - streamingDataKey, itagTypeWanted, pair.getSecond())) + new Pair<>(html5StreamingData, + new Pair<>(html5Cpn, html5StreamingUrlsPoToken)), + new Pair<>(androidStreamingData, + new Pair<>(androidCpn, androidStreamingUrlsPoToken)), + new Pair<>(iosStreamingData, + new Pair<>(iosCpn, iosStreamingUrlsPoToken))) + .flatMap(pair -> getStreamsFromStreamingDataKey( + videoId, + pair.getFirst(), + streamingDataKey, + itagTypeWanted, + pair.getSecond().getFirst(), + pair.getSecond().getSecond())) .map(streamBuilderHelper) .forEachOrdered(stream -> { if (!Stream.containSimilarStream(stream, streamList)) { @@ -1198,7 +1307,8 @@ private java.util.stream.Stream getStreamsFromStreamingDataKey( final JsonObject streamingData, final String streamingDataKey, @Nonnull final ItagItem.ItagType itagTypeWanted, - @Nonnull final String contentPlaybackNonce) { + @Nonnull final String contentPlaybackNonce, + @Nullable final String poToken) { if (streamingData == null || !streamingData.has(streamingDataKey)) { return java.util.stream.Stream.empty(); } @@ -1211,7 +1321,7 @@ private java.util.stream.Stream getStreamsFromStreamingDataKey( final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag")); if (itagItem.itagType == itagTypeWanted) { return buildAndAddItagInfoToList(videoId, formatData, itagItem, - itagItem.itagType, contentPlaybackNonce); + itagItem.itagType, contentPlaybackNonce, poToken); } } catch (final ExtractionException ignored) { // If the itag is not supported, the n parameter of HTML5 clients cannot be @@ -1227,7 +1337,8 @@ private ItagInfo buildAndAddItagInfoToList( @Nonnull final JsonObject formatData, @Nonnull final ItagItem itagItem, @Nonnull final ItagItem.ItagType itagType, - @Nonnull final String contentPlaybackNonce) throws ExtractionException { + @Nonnull final String contentPlaybackNonce, + @Nullable final String poToken) throws ExtractionException { String streamUrl; if (formatData.has("url")) { streamUrl = formatData.getString("url"); @@ -1241,9 +1352,6 @@ private ItagInfo buildAndAddItagInfoToList( streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + signature; } - // Add the content playback nonce to the stream URL - streamUrl += "&" + CPN + "=" + contentPlaybackNonce; - // Decode the n parameter if it is present // If it cannot be decoded, the stream cannot be used as streaming URLs return HTTP 403 // responses if it has not the right value @@ -1253,6 +1361,14 @@ private ItagInfo buildAndAddItagInfoToList( streamUrl = YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated( videoId, streamUrl); + // Add the content playback nonce to the stream URL + streamUrl += "&" + CPN + "=" + contentPlaybackNonce; + + // Add the poToken, if there is one + if (poToken != null) { + streamUrl += "&pot=" + poToken; + } + final JsonObject initRange = formatData.getObject("initRange"); final JsonObject indexRange = formatData.getObject("indexRange"); final String mimeType = formatData.getString("mimeType", ""); @@ -1535,10 +1651,10 @@ public List getMetaInfo() throws ParsingException { * Sets the {@link PoTokenProvider} instance to be used for fetching poTokens. * *

- * This method allows setting an implementation of {@link PoTokenProvider} which will - * be used to obtain poTokens required for YouTube player requests. These tokens are - * used by YouTube to verify the integrity of the device and may be necessary for - * playback at times. + * This method allows setting an implementation of {@link PoTokenProvider} which will be used + * to obtain poTokens required for YouTube player requests and streaming URLs. These tokens + * are used by YouTube to verify the integrity of the user's device or browser and are + * necessary for playback for several clients. *

* * @param poTokenProvider the {@link PoTokenProvider} instance to set @@ -1548,12 +1664,16 @@ public static void setPoTokenProvider(@Nullable final PoTokenProvider poTokenPro } /** - * Sets whether to force fetch the iOS player response even if the webPoTokenResult is not null. + * Sets whether to force fetch the iOS player response on livestreams. + * + *

+ * This method allows setting a flag to force the fetching of the iOS player response, which + * can be useful in scenarios where streams from the iOS player response is preferred. + *

* *

- * This method allows setting a flag to force the fetching of the iOS player response, even if a - * valid webPoTokenResult is available. This can be useful in scenarios where streams from the - * iOS player response is preferred. + * Note that at the time of writing, YouTube is rolling out a poToken requirement on this + * client. Formats from HLS manifests do not seem to be affected. *

* * @param forceFetchIosClient a boolean flag indicating whether to force fetch the iOS