Skip to content

Commit 0bd2035

Browse files
fuzhengyinfuzhengyin
committed
Merge branch 'dev' of https://github.com/fuzhengyin/NewPipeExtractor into dev
2 parents 8f32465 + f6f2724 commit 0bd2035

16 files changed

Lines changed: 189 additions & 78 deletions

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ allprojects {
2828

2929
ext {
3030
nanojsonVersion = "1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751"
31-
spotbugsVersion = "4.4.2"
31+
spotbugsVersion = "4.5.0"
3232
junitVersion = "4.13.2"
3333
}
3434
}

‎extractor/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ dependencies {
1616

1717
testImplementation "junit:junit:$junitVersion"
1818
testImplementation "com.squareup.okhttp3:okhttp:3.12.13"
19-
testImplementation 'com.google.code.gson:gson:2.8.8'
19+
testImplementation 'com.google.code.gson:gson:2.8.9'
2020
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ public static CommentsInfo getInfo(final String url) throws IOException, Extract
2424
return getInfo(NewPipe.getServiceByUrl(url), url);
2525
}
2626

27-
public static CommentsInfo getInfo(final StreamingService serviceByUrl, final String url)
27+
public static CommentsInfo getInfo(final StreamingService service, final String url)
2828
throws ExtractionException, IOException {
29-
return getInfo(serviceByUrl.getCommentsExtractor(url));
29+
return getInfo(service.getCommentsExtractor(url));
3030
}
3131

3232
public static CommentsInfo getInfo(final CommentsExtractor commentsExtractor)
@@ -63,7 +63,7 @@ public static InfoItemsPage<CommentsInfoItem> getMoreItems(
6363
final StreamingService service,
6464
final CommentsInfo commentsInfo,
6565
final Page page) throws IOException, ExtractionException {
66-
if (null == commentsInfo.getCommentsExtractor()) {
66+
if (commentsInfo.getCommentsExtractor() == null) {
6767
commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl()));
6868
commentsInfo.getCommentsExtractor().fetchPage();
6969
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
1111

1212
private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();
13-
private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)";
13+
private static final String ID_PATTERN = "((accounts|a)|(video-channels|c))/([^/?&#]*)";
1414
public static final String API_ENDPOINT = "/api/v1/";
1515

1616
public static PeertubeChannelLinkHandlerFactory getInstance() {
@@ -19,7 +19,7 @@ public static PeertubeChannelLinkHandlerFactory getInstance() {
1919

2020
@Override
2121
public String getId(String url) throws ParsingException {
22-
return Parser.matchGroup(ID_PATTERN, url, 0);
22+
return fixId(Parser.matchGroup(ID_PATTERN, url, 0));
2323
}
2424

2525
@Override
@@ -31,7 +31,7 @@ public String getUrl(String id, List<String> contentFilters, String searchFilter
3131
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl)
3232
throws ParsingException {
3333
if (id.matches(ID_PATTERN)) {
34-
return baseUrl + "/" + id;
34+
return baseUrl + "/" + fixId(id);
3535
} else {
3636
// This is needed for compatibility with older versions were we didn't support video channels yet
3737
return baseUrl + "/accounts/" + id;
@@ -40,6 +40,28 @@ public String getUrl(String id, List<String> contentFilter, String sortFilter, S
4040

4141
@Override
4242
public boolean onAcceptUrl(String url) {
43-
return url.contains("/accounts/") || url.contains("/video-channels/");
43+
return url.contains("/accounts/") || url.contains("/a/")
44+
|| url.contains("/video-channels/") || url.contains("/c/");
45+
}
46+
47+
/**
48+
* Fix id
49+
*
50+
* <p>
51+
* a/:accountName and c/:channelName ids are supported
52+
* by the PeerTube web client (>= v3.3.0)
53+
* but not by the API.
54+
* </p>
55+
*
56+
* @param id the id to fix
57+
* @return the fixed id
58+
*/
59+
private String fixId(String id) {
60+
if (id.startsWith("a/")) {
61+
id = "accounts" + id.substring(1);
62+
} else if (id.startsWith("c/")) {
63+
id = "video-channels" + id.substring(1);
64+
}
65+
return id;
4466
}
4567
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
1212

1313
private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory();
14-
private static final String ID_PATTERN = "/videos/(watch/)?([^/?&#]*)";
1514
private static final String COMMENTS_ENDPOINT = "/api/v1/videos/%s/comment-threads";
1615

1716
public static PeertubeCommentsLinkHandlerFactory getInstance() {
@@ -20,12 +19,12 @@ public static PeertubeCommentsLinkHandlerFactory getInstance() {
2019

2120
@Override
2221
public String getId(String url) throws ParsingException, IllegalArgumentException {
23-
return Parser.matchGroup(ID_PATTERN, url, 2);
22+
return PeertubeStreamLinkHandlerFactory.getInstance().getId(url); // the same id is needed
2423
}
2524

2625
@Override
2726
public boolean onAcceptUrl(final String url) throws FoundAdException {
28-
return url.contains("/videos/");
27+
return url.contains("/videos/") || url.contains("/w/");
2928
}
3029

3130
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
1212

1313
private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory();
14-
private static final String ID_PATTERN = "/videos/watch/playlist/([^/?&#]*)";
14+
private static final String ID_PATTERN = "(/videos/watch/playlist/|/w/p/)([^/?&#]*)";
1515

1616
public static PeertubePlaylistLinkHandlerFactory getInstance() {
1717
return instance;
@@ -30,7 +30,7 @@ public String getUrl(String id, List<String> contentFilters, String sortFilter,
3030

3131
@Override
3232
public String getId(String url) throws ParsingException {
33-
return Parser.matchGroup1(ID_PATTERN, url);
33+
return Parser.matchGroup(ID_PATTERN, url, 2);
3434
}
3535

3636
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,13 @@
99
public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
1010

1111
private static final PeertubeStreamLinkHandlerFactory instance = new PeertubeStreamLinkHandlerFactory();
12-
private static final String ID_PATTERN = "/videos/(watch/|embed/)?([^/?&#]*)";
12+
private static final String ID_PATTERN = "(/w/|(/videos/(watch/|embed/)?))(?!p/)([^/?&#]*)";
13+
// we exclude p/ because /w/p/ is playlist, not video
1314
public static final String VIDEO_API_ENDPOINT = "/api/v1/videos/";
15+
16+
// From PeerTube 3.3.0, the default path is /w/.
17+
// We still use /videos/watch/ for compatibility reasons:
18+
// /videos/watch/ is still accepted by >=3.3.0 but /w/ isn't by <3.3.0
1419
private static final String VIDEO_PATH = "/videos/watch/";
1520

1621
private PeertubeStreamLinkHandlerFactory() {
@@ -32,7 +37,7 @@ public String getUrl(String id, String baseUrl) {
3237

3338
@Override
3439
public String getId(String url) throws ParsingException, IllegalArgumentException {
35-
return Parser.matchGroup(ID_PATTERN, url, 2);
40+
return Parser.matchGroup(ID_PATTERN, url, 4);
3641
}
3742

3843
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141

4242
public class SoundcloudParsingHelper {
4343
private static final String HARDCODED_CLIENT_ID =
44-
"yoxLvaFlJ3V5LbNCt53Cwvw5KXKKxWfn"; // Updated on 01/10/21
44+
"nGKlrpy2IotLQ0QGwBOmIgSFayis6H4e"; // Updated on 04/11/21
4545
private static String clientId;
4646
public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/";
4747

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,10 @@ public static boolean isInvidioURL(@Nonnull final URL url) {
162162
|| host.equalsIgnoreCase("y.com.cm");
163163
}
164164

165+
public static boolean isY2ubeURL(@Nonnull final URL url) {
166+
return url.getHost().equalsIgnoreCase("y2u.be");
167+
}
168+
165169
/**
166170
* Parses the duration string of the video expecting ":" or "." as separators
167171
*

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 38 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -337,19 +337,33 @@ public long getViewCount() throws ParsingException {
337337
@Override
338338
public long getLikeCount() throws ParsingException {
339339
assertPageFetched();
340-
String likesString = "";
340+
String likesString = null;
341341
try {
342-
try {
343-
likesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
344-
.getObject("sentimentBarRenderer").getString("tooltip").split("/")[0];
345-
} catch (final NullPointerException e) {
342+
likesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
343+
.getObject("sentimentBarRenderer").getString("tooltip");
344+
if (likesString != null && likesString.contains("/")) {
345+
likesString = likesString.split("/")[0];
346+
} else {
347+
likesString = getVideoPrimaryInfoRenderer()
348+
.getObject("videoActions")
349+
.getObject("menuRenderer")
350+
.getArray("topLevelButtons")
351+
.getObject(0)
352+
.getObject("toggleButtonRenderer")
353+
.getObject("defaultText")
354+
.getObject("accessibility")
355+
.getObject("accessibilityData")
356+
.getString("label");
357+
}
358+
359+
if (likesString == null) {
346360
// If this kicks in our button has no content and therefore ratings must be disabled
347361
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
348-
throw new ParsingException(
349-
"Ratings are enabled even though the like button is missing", e);
362+
throw new ParsingException("Ratings are enabled even though the like button is missing");
350363
}
351364
return -1;
352365
}
366+
353367
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
354368
} catch (final NumberFormatException nfe) {
355369
throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer",
@@ -366,29 +380,28 @@ public long getLikeCount() throws ParsingException {
366380
public long getDislikeCount() throws ParsingException {
367381
assertPageFetched();
368382

369-
String dislikesString = "";
370383
try {
371-
try {
372-
dislikesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
373-
.getObject("sentimentBarRenderer").getString("tooltip").split("/")[1];
374-
} catch (final NullPointerException e) {
375-
// If this kicks in our button has no content and therefore ratings must be disabled
376-
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
377-
throw new ParsingException(
378-
"Ratings are enabled even though the dislike button is missing", e);
384+
String dislikesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
385+
.getObject("sentimentBarRenderer").getString("tooltip");
386+
if (dislikesString != null && dislikesString.contains("/")) {
387+
dislikesString = dislikesString.split("/")[1];
388+
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
389+
} else {
390+
// Calculate dislike with average rating and like count
391+
long likes = getLikeCount();
392+
double averageRating = playerResponse.getObject("videoDetails").getDouble("averageRating");
393+
394+
if (likes != -1 && averageRating > 1) {
395+
// If averageRating can't be gathered, it will be 0,
396+
// but we also can't divide by 0 so we need > 1
397+
return Math.round(likes * ((5 - averageRating) / (averageRating - 1)));
379398
}
380-
return -1;
381399
}
382-
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
383-
} catch (final NumberFormatException nfe) {
384-
throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer",
385-
nfe);
386400
} catch (final Exception e) {
387-
if (getAgeLimit() == NO_AGE_LIMIT) {
388-
throw new ParsingException("Could not get dislike count", e);
389-
}
390-
return -1;
391401
}
402+
// Silently fail as YouTube is "gradually rolling out" removing dislike count
403+
// https://blog.youtube/news-and-events/update-to-youtube/
404+
return -1;
392405
}
393406

394407
@Nonnull

0 commit comments

Comments
 (0)