This question came up on Stack Overflow, which I liked the look of. I’ve been doing Selenium development in .NET framework for work recently, but thought I’d give this JS based question a crack. When I got down to it, I realised that it’s very similar, and the JS is pretty easy to process.

So here’s my solution for processing each tweet in a feed, and finding all links with the media class within the tweet. I don’t have a reason for this at the moment, but wanted to post here so I don’t forget/lose it. I hope it comes in handy for someone

//Helpful functions for finding elements by xPath
getElementsByXpath = function (path, doc = document) {
    return document.evaluate(path, doc, null, XPathResult.ANY_TYPE, null);
};
getElementByXpath = function (path, doc = document) {
    return document.evaluate(path, doc, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
};

console.log("Starting...");

var tweetsXpath = "//li[contains(@id,'stream-item-tweet')]";
var linksXpath = "//a[contains(@class,'twitter-timeline-link') and contains(@class,'media')]";
var tweets = getElementsByXpath(tweetsXpath);

console.log("Tweets: " +tweets); //tweets is an XPathResult object. We can iterate over this.

var tweet = tweets.iterateNext(); //get first tweet
while(tweet != null){
    var links = getElementsByXpath(linksXpath, tweet);
    var link = links.iterateNext(); //get first link
    
    while(link != null) {
        var linkURL = link.getAttribute("href");
        //do something with linkURL here
        console.log("Found media link in tweet - URL: " + linkURL);

        link = links.iterateNext(); //move to next link
    }
    tweet = tweets.iterateNext(); //move to next tweet
}
console.log("Done");