Skip to content

Commit c953e23

Browse files
FireMasterKFireMasterK
authored
Merge pull request #968 from AudricV/yt-support-no-video-info-renderers-for-streams
[YouTube] Support lack of video info renderers for streams
2 parents 2211a24 + 6a2c680 commit c953e23

File tree

9 files changed

+1754
-73
lines changed

9 files changed

+1754
-73
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

+56-73
Original file line numberDiff line numberDiff line change
@@ -204,45 +204,48 @@ public String getTextualUploadDate() throws ParsingException {
204204
return null;
205205
}
206206

207-
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
208-
.startsWith("Premiered")) {
209-
final String time = getTextFromObject(
210-
getVideoPrimaryInfoRenderer().getObject("dateText")).substring(13);
211-
212-
try { // Premiered 20 hours ago
213-
final TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(
214-
Localization.fromLocalizationCode("en"));
215-
final OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime();
216-
return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime);
217-
} catch (final Exception ignored) {
218-
}
207+
final String videoPrimaryInfoRendererDateText =
208+
getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"));
209+
210+
if (videoPrimaryInfoRendererDateText != null) {
211+
if (videoPrimaryInfoRendererDateText.startsWith("Premiered")) {
212+
final String time = videoPrimaryInfoRendererDateText.substring(13);
213+
214+
try { // Premiered 20 hours ago
215+
final TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(
216+
Localization.fromLocalizationCode("en"));
217+
final OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime();
218+
return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime);
219+
} catch (final Exception ignored) {
220+
}
219221

220-
try { // Premiered Feb 21, 2020
221-
final LocalDate localDate = LocalDate.parse(time,
222-
DateTimeFormatter.ofPattern("MMM dd, yyyy", Locale.ENGLISH));
223-
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
224-
} catch (final Exception ignored) {
222+
try { // Premiered Feb 21, 2020
223+
final LocalDate localDate = LocalDate.parse(time,
224+
DateTimeFormatter.ofPattern("MMM dd, yyyy", Locale.ENGLISH));
225+
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
226+
} catch (final Exception ignored) {
227+
}
228+
229+
try { // Premiered on 21 Feb 2020
230+
final LocalDate localDate = LocalDate.parse(time,
231+
DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
232+
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
233+
} catch (final Exception ignored) {
234+
}
225235
}
226236

227-
try { // Premiered on 21 Feb 2020
228-
final LocalDate localDate = LocalDate.parse(time,
237+
try {
238+
// TODO: this parses English formatted dates only, we need a better approach to
239+
// parse the textual date
240+
final LocalDate localDate = LocalDate.parse(videoPrimaryInfoRendererDateText,
229241
DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
230242
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
231-
} catch (final Exception ignored) {
243+
} catch (final Exception e) {
244+
throw new ParsingException("Could not get upload date", e);
232245
}
233246
}
234247

235-
try {
236-
// TODO: this parses English formatted dates only, we need a better approach to parse
237-
// the textual date
238-
final LocalDate localDate = LocalDate.parse(getTextFromObject(
239-
getVideoPrimaryInfoRenderer().getObject("dateText")),
240-
DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
241-
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
242-
} catch (final Exception e) {
243-
throw new ParsingException("Could not get upload date", e);
244-
}
245-
248+
throw new ParsingException("Could not get upload date");
246249
}
247250

248251
@Override
@@ -565,19 +568,13 @@ public boolean isUploaderVerified() throws ParsingException {
565568
public String getUploaderAvatarUrl() throws ParsingException {
566569
assertPageFetched();
567570

568-
String url = null;
569-
570-
try {
571-
url = getVideoSecondaryInfoRenderer()
572-
.getObject("owner")
573-
.getObject("videoOwnerRenderer")
574-
.getObject("thumbnail")
575-
.getArray("thumbnails")
576-
.getObject(0)
577-
.getString("url");
578-
} catch (final ParsingException ignored) {
579-
// Age-restricted videos cause a ParsingException here
580-
}
571+
final String url = getVideoSecondaryInfoRenderer()
572+
.getObject("owner")
573+
.getObject("videoOwnerRenderer")
574+
.getObject("thumbnail")
575+
.getArray("thumbnails")
576+
.getObject(0)
577+
.getString("url");
581578

582579
if (isNullOrEmpty(url)) {
583580
if (ageLimit == NO_AGE_LIMIT) {
@@ -1212,54 +1209,40 @@ private String deobfuscateSignature(final String obfuscatedSig) throws ParsingEx
12121209
// Utils
12131210
//////////////////////////////////////////////////////////////////////////*/
12141211

1215-
private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException {
1212+
@Nonnull
1213+
private JsonObject getVideoPrimaryInfoRenderer() {
12161214
if (videoPrimaryInfoRenderer != null) {
12171215
return videoPrimaryInfoRenderer;
12181216
}
12191217

1220-
final JsonArray contents = nextResponse.getObject("contents")
1221-
.getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
1222-
.getArray("contents");
1223-
JsonObject theVideoPrimaryInfoRenderer = null;
1224-
1225-
for (final Object content : contents) {
1226-
if (((JsonObject) content).has("videoPrimaryInfoRenderer")) {
1227-
theVideoPrimaryInfoRenderer = ((JsonObject) content)
1228-
.getObject("videoPrimaryInfoRenderer");
1229-
break;
1230-
}
1231-
}
1232-
1233-
if (isNullOrEmpty(theVideoPrimaryInfoRenderer)) {
1234-
throw new ParsingException("Could not find videoPrimaryInfoRenderer");
1235-
}
1236-
1237-
videoPrimaryInfoRenderer = theVideoPrimaryInfoRenderer;
1238-
return theVideoPrimaryInfoRenderer;
1218+
videoPrimaryInfoRenderer = getVideoInfoRenderer("videoPrimaryInfoRenderer");
1219+
return videoPrimaryInfoRenderer;
12391220
}
12401221

12411222
@Nonnull
1242-
private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
1223+
private JsonObject getVideoSecondaryInfoRenderer() {
12431224
if (videoSecondaryInfoRenderer != null) {
12441225
return videoSecondaryInfoRenderer;
12451226
}
12461227

1247-
videoSecondaryInfoRenderer = nextResponse
1248-
.getObject("contents")
1228+
videoSecondaryInfoRenderer = getVideoInfoRenderer("videoSecondaryInfoRenderer");
1229+
return videoSecondaryInfoRenderer;
1230+
}
1231+
1232+
@Nonnull
1233+
private JsonObject getVideoInfoRenderer(@Nonnull final String videoRendererName) {
1234+
return nextResponse.getObject("contents")
12491235
.getObject("twoColumnWatchNextResults")
12501236
.getObject("results")
12511237
.getObject("results")
12521238
.getArray("contents")
12531239
.stream()
12541240
.filter(JsonObject.class::isInstance)
12551241
.map(JsonObject.class::cast)
1256-
.filter(content -> content.has("videoSecondaryInfoRenderer"))
1257-
.map(content -> content.getObject("videoSecondaryInfoRenderer"))
1242+
.filter(content -> content.has(videoRendererName))
1243+
.map(content -> content.getObject(videoRendererName))
12581244
.findFirst()
1259-
.orElseThrow(
1260-
() -> new ParsingException("Could not find videoSecondaryInfoRenderer"));
1261-
1262-
return videoSecondaryInfoRenderer;
1245+
.orElse(new JsonObject());
12631246
}
12641247

12651248
@Nonnull

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

+63
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,69 @@ public static void setUp() throws Exception {
430430
// @formatter:on
431431
}
432432

433+
public static class NoVisualMetadataVideoTest extends DefaultStreamExtractorTest {
434+
// Video without visual metadata on YouTube clients (video title, upload date, channel name,
435+
// comments, ...)
436+
private static final String ID = "An8vtD1FDqs";
437+
private static final String URL = BASE_URL + ID;
438+
private static StreamExtractor extractor;
439+
440+
@BeforeAll
441+
public static void setUp() throws Exception {
442+
YoutubeTestsUtils.ensureStateless();
443+
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "noVisualMetadata"));
444+
extractor = YouTube.getStreamExtractor(URL);
445+
extractor.fetchPage();
446+
}
447+
448+
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
449+
@Override public String expectedUploaderName() { return "Makani"; }
450+
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC-iMZJ8NppwT2fLwzFWJKOQ"; }
451+
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("Makani", "prototype", "rotors"); }
452+
@Override public long expectedLength() { return 175; }
453+
@Override public long expectedViewCountAtLeast() { return 88_000; }
454+
@Nullable @Override public String expectedUploadDate() { return "2017-05-16 00:00:00.000"; }
455+
@Nullable @Override public String expectedTextualUploadDate() { return "2017-05-16"; }
456+
@Override public long expectedLikeCountAtLeast() { return -1; }
457+
@Override public long expectedDislikeCountAtLeast() { return -1; }
458+
@Override public StreamExtractor extractor() { return extractor; }
459+
@Override public StreamingService expectedService() { return YouTube; }
460+
@Override public String expectedName() { return "Makani’s first commercial-scale energy kite"; }
461+
@Override public String expectedId() { return "An8vtD1FDqs"; }
462+
@Override public String expectedUrlContains() { return BASE_URL + ID; }
463+
@Override public String expectedOriginalUrlContains() { return URL; }
464+
@Override public String expectedCategory() { return "Science & Technology"; }
465+
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
466+
@Override public List<String> expectedTags() {
467+
return Arrays.asList("Makani", "Moonshot", "Moonshot Factory", "Prototyping",
468+
"california", "california wind", "clean", "clean energy", "climate change",
469+
"climate crisis", "energy", "energy kite", "google", "google x", "green",
470+
"green energy", "kite", "kite power", "kite power solutions",
471+
"kite power systems", "makani power", "power", "renewable", "renewable energy",
472+
"renewable energy engineering", "renewable energy projects",
473+
"renewable energy sources", "renewables", "solutions", "tech", "technology",
474+
"turbine", "wind", "wind energy", "wind power", "wind turbine", "windmill");
475+
}
476+
477+
@Test
478+
@Override
479+
public void testSubscriberCount() {
480+
assertThrows(ParsingException.class, () -> extractor.getUploaderSubscriberCount());
481+
}
482+
483+
@Test
484+
@Override
485+
public void testLikeCount() {
486+
assertThrows(ParsingException.class, () -> extractor.getLikeCount());
487+
}
488+
489+
@Test
490+
@Override
491+
public void testUploaderAvatarUrl() {
492+
assertThrows(ParsingException.class, () -> extractor.getUploaderAvatarUrl());
493+
}
494+
}
495+
433496
public static class UnlistedTest {
434497
private static YoutubeStreamExtractor extractor;
435498

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/stream/noVisualMetadata/generated_mock_0.json

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
{
2+
"request": {
3+
"httpMethod": "GET",
4+
"url": "https://www.youtube.com/iframe_api",
5+
"headers": {
6+
"Accept-Language": [
7+
"en-GB, en;q\u003d0.9"
8+
]
9+
},
10+
"localization": {
11+
"languageCode": "en",
12+
"countryCode": "GB"
13+
}
14+
},
15+
"response": {
16+
"responseCode": 200,
17+
"responseMessage": "",
18+
"responseHeaders": {
19+
"alt-svc": [
20+
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
21+
],
22+
"cache-control": [
23+
"private, max-age\u003d0"
24+
],
25+
"content-type": [
26+
"text/javascript; charset\u003dutf-8"
27+
],
28+
"cross-origin-opener-policy-report-only": [
29+
"same-origin; report-to\u003d\"youtube_main\""
30+
],
31+
"cross-origin-resource-policy": [
32+
"cross-origin"
33+
],
34+
"date": [
35+
"Fri, 04 Nov 2022 18:36:38 GMT"
36+
],
37+
"expires": [
38+
"Fri, 04 Nov 2022 18:36:38 GMT"
39+
],
40+
"p3p": [
41+
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
42+
],
43+
"permissions-policy": [
44+
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
45+
],
46+
"report-to": [
47+
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
48+
],
49+
"server": [
50+
"ESF"
51+
],
52+
"set-cookie": [
53+
"YSC\u003dUBx6tMGNmRg; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
54+
"VISITOR_INFO1_LIVE\u003dvY4W1Ai6Us0; Domain\u003d.youtube.com; Expires\u003dWed, 03-May-2023 18:36:38 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
55+
"CONSENT\u003dPENDING+815; expires\u003dSun, 03-Nov-2024 18:36:38 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
56+
],
57+
"strict-transport-security": [
58+
"max-age\u003d31536000"
59+
],
60+
"x-content-type-options": [
61+
"nosniff"
62+
],
63+
"x-frame-options": [
64+
"SAMEORIGIN"
65+
],
66+
"x-xss-protection": [
67+
"0"
68+
]
69+
},
70+
"responseBody": "var scriptUrl \u003d \u0027https:\\/\\/www.youtube.com\\/s\\/player\\/03bec62d\\/www-widgetapi.vflset\\/www-widgetapi.js\u0027;try{var ttPolicy\u003dwindow.trustedTypes.createPolicy(\"youtube-widget-api\",{createScriptURL:function(x){return x}});scriptUrl\u003dttPolicy.createScriptURL(scriptUrl)}catch(e){}var YT;if(!window[\"YT\"])YT\u003d{loading:0,loaded:0};var YTConfig;if(!window[\"YTConfig\"])YTConfig\u003d{\"host\":\"https://www.youtube.com\"};\nif(!YT.loading){YT.loading\u003d1;(function(){var l\u003d[];YT.ready\u003dfunction(f){if(YT.loaded)f();else l.push(f)};window.onYTReady\u003dfunction(){YT.loaded\u003d1;for(var i\u003d0;i\u003cl.length;i++)try{l[i]()}catch(e$0){}};YT.setConfig\u003dfunction(c){for(var k in c)if(c.hasOwnProperty(k))YTConfig[k]\u003dc[k]};var a\u003ddocument.createElement(\"script\");a.type\u003d\"text/javascript\";a.id\u003d\"www-widgetapi-script\";a.src\u003dscriptUrl;a.async\u003dtrue;var c\u003ddocument.currentScript;if(c){var n\u003dc.nonce||c.getAttribute(\"nonce\");if(n)a.setAttribute(\"nonce\",n)}var b\u003d\ndocument.getElementsByTagName(\"script\")[0];b.parentNode.insertBefore(a,b)})()};\n",
71+
"latestUrl": "https://www.youtube.com/iframe_api"
72+
}
73+
}

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/stream/noVisualMetadata/generated_mock_1.json

+71
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)