Is there a javascript to remove duplicate videos in a Youtube playlist?

Question

Since YouTube discontinued the remove duplicates button from its interface (at about the same time they discontinued their non-polymer interface), users are at a loss deduping their playlists. I've looked everywhere for tool to do that, to no avail. I found this script to remove all videos from a playlist, and wonder whether it can be modified (by someone more knowledgeable of javascript than I) to remove only duplicate videos:

setInterval(function () {
  document.querySelector('#primary button[aria-label="Action menu"]').click();
  var things = document.evaluate(
    '//span[contains(text(),"Remove from")]',
    document,
    null,
    XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,
    null
  );
  for (var i = 0; i < things.snapshotLength; i++) {
    things.snapshotItem(i).click();
  }
}, 1000);

Answer 1

There is actually still (as of 2021-11-07) a kind of official way to create a new duplicate-free copy of a playlist:

1. Open Youtube Music and create a playlist https://music.youtube.com/library/playlists (playlist needs to be created on youtube music, otherwise the playlist will not be visible on youtube music...)

2. Open the playlist from which you want to remove duplicates on regular youtube, eg: https://www.youtube.com/playlist?list=PLlrATfBNZ98dudnM48yfGUldqGD0S4FFb

3. Use the menu of the first video (three dots on the right side of the video) and choose "save to playlist" and save the first video of your playlist to the newly created youtube music playlist (which was create in step 1)

4. Replace www in your url with music to get to the youtube music view of your old playlist, e.g: https://music.youtube.com/playlist?list=PLlrATfBNZ98dudnM48yfGUldqGD0S4FFb

5. Use the menu (three dots) and select "Add to Playlist" to add your old playlist to your new (in step 1 created) playlist. Because the new playlist already contains a video of your old playlist (added in step 3) you now get a question if you want to skip duplicates, so...

6. ... you select "Skip Duplicates". This seems to not only skip videos that were originally already contained in your new playlist, but also only adds each video of the old playlist only once and thereby removes duplicates.

---

Alternative: Javascript solution:

It's not really nice, because it only seems to work with a sleep after each button press, but this worked for me:

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

async function removeduplicates() {
    let titles = document.querySelectorAll('#primary #video-title')
    let href_pattern = RegExp('https?://www\\.youtube\\.com/watch\\?v=[^&]*&list=[^&]*&index=')
    titles = Array.from(titles).filter(t => href_pattern.test(t.href))
    let lastid='';
    for (let i = 0; i < titles.length; i++) {
        let id = titles[i].href.match(/\\?v=([^&]+)/)[1]
        if(id == lastid){
            titles[i].focus()
            titles[i].parentElement.parentElement.parentElement.parentElement.parentElement.querySelector('button[aria-label="Action menu"]').click()
            await sleep(100)
            var things = document.evaluate(
                '//span[contains(text(),"Remove from")]',
                document,
                null,
                XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,
                null
            );
            await sleep(300)
            for (var j = 0; j < things.snapshotLength; j++) {
                things.snapshotItem(j).click();
            }
            console.log(titles[i].innerText)
        }
        lastid=id;
    }
}

removeduplicates()

Also note, that it will only work if your language is set to english, because the remove button is found via the string "Remove from".

To run this, you need to be on the playlist page (e.g. https://www.youtube.com/playlist?list=PLlrATfBNZ98dudnM48yfGUldqGD0S4FFb) of the playlist, from which you want to remove duplicates. Then press F12, paste the code into the javascript console and press enter.

Note: Duplicates are only removed if two identical videos follow each other, so you need to sort the playlist by e.g. publishing date of video.

Note2: Only the videos currently loaded into the browser window are considered, so you need to scroll down until the end of the playlist, such that all videos are actually loaded.

Answer 2

I actually took the script used and improved on it a little. This new script doesn't require sorting the playlist at all

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}
function $$(selector,context=document.documentElement){return[...context?.querySelectorAll?.(selector) ?? []]}

async function removeduplicates() {
    let titles = $$("#primary a#video-title")
    .filter((i,j,k)=>k.
    findIndex(k=>k.href.includes(Object.fromEntries(new URLSearchParams(i.href))['https://www.youtube.com/watch?v']))!==j)
    for (let i = 0; i < titles.length; i++) {
            titles[i].focus()
            titles[i].closest("#contents > *").querySelector('button[aria-label="Action menu"]').click()
            await sleep(100)
            var things = document.evaluate(
                '//span[contains(text(),"Remove from")]',
                document,
                null,
                XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,
                null
            );
            await sleep(300)
            for (var j = 0; j < things.snapshotLength; j++) {
                things.snapshotItem(j).click();
            }
            console.log(titles[i].innerText)
        }
    }

removeduplicates()

Instead of checking video titles, it checks the hrefs for duplicates, isolates only the duplicate videos, and removes the duplicates. Tested it myself, works like a charm

Answer 3

I dont have enough reputation to comment, but @André Levy , heres the same code where I formatted it and added semicolons. It looks pretty bad but works a charm as a bookmarklet. Just copy paste it into your link:

javascript:function sleep(ms) {  return new Promise(resolve => setTimeout(resolve, ms));}function $$(selector,context=document.documentElement){return[...context?.querySelectorAll?.(selector) ?? []]};async function removeduplicates() {    let titles = $$("#primary a#video-title")    .filter((i,j,k)=>k.    findIndex(k=>k.href.includes(Object.fromEntries(new URLSearchParams(i.href))['https://www.youtube.com/watch?v']))!==j);    for (let i = 0; i < titles.length; i++) {            titles[i].focus();            titles[i].closest("#contents > *").querySelector('button[aria-label="Action menu"]').click();            await sleep(100);            var things = document.evaluate(                '//span[contains(text(),"Remove from")]',                document,                null,                XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,                null            );            await sleep(300);            for (var j = 0; j < things.snapshotLength; j++) {                things.snapshotItem(j).click();            }            console.log(titles[i].innerText);        }    }removeduplicates();