From: Brian Flowers Date: Sat, 24 Apr 2021 02:53:00 +0000 (-0400) Subject: Initial Commit X-Git-Url: http://git.slightlycyberpunk.com%2C%20git.slightlycyberpunk.com/git/?a=commitdiff_plain;h=ad3f4c9e4abd682ef41860f83b851e4f302154b0;p=mastodon-tools.git Initial Commit --- ad3f4c9e4abd682ef41860f83b851e4f302154b0 diff --git a/forecast.sh b/forecast.sh new file mode 100755 index 0000000..54f4d9f --- /dev/null +++ b/forecast.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +# MESSAGES: +# SUBSCRIBE [frequency] [zip] [time] -- SUBSCRIBE daily 02895 7:00 +# UNSUBSCRIBE + +# TODO: +# Remove hardcoded URLs and user info, put variables somewhere common +# Maybe rewrite this to not be a shell script? + +# Fill in the app ID from openweathermap +APPID="" + +# Fill in the token below from your Mastodon developer UI! +setsubs() { +token="" +maxid="`cat weather.maxid`" + +curl -X GET \ + -H "Authorization: Bearer ${token}" \ + https://mastodon.slightlycyberpunk.com/api/v1/conversations?since_id=$maxid |\ + sed 's/}},{"id"/}}\n{"id"/g' |\ + grep -v '"account":{"id":"161","username":"forecast"' |\ + sed 's/[{}]//g' |\ + awk -v FS=',' -v OFS="|" '{ + for(i=1;i<=NF;i++){ + split($i,a,":"); + if(a[1] == "\"content\"") { + content=$i; + } else if(a[1] == "\"username\"" && a[2] != "\"forecast\"") { + user=a[2]; + } else if(a[1] == "\"last_status\"") { + id=a[3]; + } + }} + END { print user,content,id }' |\ + sort -u | grep -v '"username":"forecast"' |\ + while read line; do + if [ -z "$line" -o "$line" == "||" ]; then + continue + fi + user="`echo "$line" | cut -d'|' -f1 | cut -d'"' -f2`" + id="`echo "$line" | cut -d'|' -f3 | cut -d'"' -f2`" + subs="`echo "$line" | cut -d'|' -f2 | grep "[^a-zA-Z]SUBSCRIBE" | sed 's/.*\([^A-Za-z]SUBSCRIBE [a-zA-Z]* [0-9]* [0-9]*:[0-9]*\).*/\1/g'`" + unsubs="`echo "$line" | cut -d'|' -f2 | grep "UNSUBSCRIBE"`" + greetings="`echo "$line" | cut -d'|' -f2 | grep "Hello"`" + + if [ "$id" -gt "$maxid" ]; then + echo "$id" > weather.maxid + fi + + if [ ! -z "$unsubs" ]; then + echo "UNSUBSCRIBING $user ($unsubs)" + curl -X POST \ + -F 'Idempotency-Key=abc' \ + -H "Authorization: Bearer ${token}" \ + -F "status=ALL SUBSCRIPTIONS CANCELLED @${user}. If this doesn't work, please contact @admin@mastodon.slightlycyberpunk.com" \ + -F 'visibility=direct' \ + https://mastodon.slightlycyberpunk.com/api/v1/statuses + cat weather.subs | grep -v "^$user|" > weather.subs.2 + mv weather.subs.2 weather.subs + fi + + if [ "`cat weather.subs | grep "$user|$subs" | wc -l`" -eq 0 -a ! -z "$subs" ]; then + echo "SUBSCRIBING $user" + echo "$user|$subs" >> weather.subs + + curl -X POST \ + -F 'Idempotency-Key=abc' \ + -H "Authorization: Bearer ${token}" \ + -F "status=SUBSCRIPTION CONFIRMED @$user" \ + -F 'visibility=direct' \ + https://mastodon.slightlycyberpunk.com/api/v1/statuses + fi + + if [ ! -z "$greetings" ]; then + echo "GEETING $user" + + curl -X POST \ + -F 'Idempotency-Key=abc' \ + -H "Authorization: Bearer ${token}" \ + -F "status=Greetings, @${user}! To subscribe for daily forecasts, send as a direct message: +SUBSCRIBE daily [zipcode] [time] + +Currently only sending daily forecasts +Zipcode must be a 5-digit US zipcode +Time should be the time to send the forecast, in 24-hour EST format, as an even hour (ie, must end in :00) + +To unsubscribe from all forecasts, send as a direct message: +UNSUBSCRIBE" \ + -F 'visibility=direct' \ + https://mastodon.slightlycyberpunk.com/api/v1/statuses 2>/dev/null + fi + + done +} + +while [ 0 ]; do + setsubs + cat weather.subs | sed 's/[ ]*|[ ]*/|/g' | grep "|SUBSCRIBE" | while read line; do + user="`echo "$line" | cut -d'|' -f1`" + type="`echo "$line" | cut -d'|' -f2 | cut -d' ' -f2`" + zip="`echo "$line" | cut -d'|' -f2 | cut -d' ' -f3`" + arg="`echo "$line" | cut -d'|' -f2 | cut -d' ' -f4`" + + if [ "$type" == "daily" -a "$arg" == "`date +%H:00`" ]; then + LOCATION="$zip" + forecast="`curl --connect-timeout 20 \ + "http://api.openweathermap.org/data/2.5/forecast/daily?APPID=$APPID&q=$LOCATION&mode=xml&units=imperial&cnt=1" 2>/dev/null |\ + sed 's/>\n/dev/null | uniq | grep -v waitFor | grep -v TypeError | uniq | while read line; do + if [ ! -z "`echo "$line" | grep "^CHECK"`" ]; then + continue + fi + if [ "$line" == "-------------------------" ]; then + if [ -z "`echo "$content" | grep -v '^REF'`" ]; then + continue + fi + + # Extract source reference + ref="`echo "$content" | grep '^REF' | tail -1 | cut -d':' -f2-`" + refline="Original post on Twitter: ${ref}" + + # Extract video links + vid="`echo "$content" | grep '^VIDEO:' | tail -1 | cut -d':' -f2- | sed 's/^ *//g'`" + media_ids="" + if [ ! -z "$vid" ]; then + echo "Pulling video..." + ffmpeg -i "$vid" -c copy -bsf:a aac_adtstoasc output.mp4 >/dev/null 2>&1 + echo "Downloaded." + curl -X POST \ + -H "Authorization: Bearer ${token}" \ + -F 'file=@output.mp4' \ + https://mastodon.slightlycyberpunk.com/api/v1/media > vjson + mediaid="`cat vjson | sed 's/.*"id":"\([0-9]*\)".*/\1/g'`" + media_ids="media_ids[]=$mediaid" + echo "IDs: $media_ids" + fi + + while read img; do + if [ ! -z "$img" ]; then + format="`echo "$img" | sed 's/.*format=\([a-zA-Z0-9]*\)&.*/\1/g'`" + echo "Pulling image.${format}..." + wget -O "image.${format}" "$img" #>/dev/null 2>&1 + curl -X POST \ + -H "Authorization: Bearer ${token}" \ + -F "file=@image.${format}" \ + https://mastodon.slightlycyberpunk.com/api/v1/media > vjson + mediaid="`cat vjson | sed 's/.*"id":"\([0-9]*\)".*/\1/g'`" + if [ ! -z "$media_ids" ]; then + media_ids="$media_ids&media_ids[]=$mediaid" + else + media_ids="media_ids[]=$mediaid" + fi + echo "IDs: $media_ids" + fi + done < <(echo "$content" | grep "^IMAGE:" | cut -d':' -f2- | sed 's/^ *//g') + + echo "IDs: $media_ids" + + while read link; do + content="${content} + +${link}" + done < <(echo "$content" | grep "^LINK:" | cut -d':' -f2- | sed 's/^ *//g' | cut -d'|' -f1 | cut -d'?' -f1) + +echo "CONTENT: $content" +echo "REFLINE: $refline" + + fullcontent="`echo "$content" | grep -v '^REF' | grep -v '^VIDEO' | grep -v '^IMAGE:' | grep -v '^LINK:' | cut -c1-430 | sed 's/"/\\\"/g'` + +$refline" + +echo "POSTING:" +echo "${fullcontent}" + + if [ ! -z "$media_ids" ]; then + curl -X POST \ + -F 'Idempotency-Key=abc' \ + -H "Authorization: Bearer ${token}" \ + -F "status=\"${fullcontent}\"" \ + -F "$media_ids" \ + -F 'visibility=private' \ + https://mastodon.slightlycyberpunk.com/api/v1/statuses + + if [ -f output.mp4 ]; then + rm output.mp4 + elif [ -f image.* ]; then + rm image.* + fi + else + curl -X POST \ + -F 'Idempotency-Key=abc' \ + -H "Authorization: Bearer ${token}" \ + -F "status=\"${fullcontent}\"" \ + -F 'visibility=private' \ + https://mastodon.slightlycyberpunk.com/api/v1/statuses + fi + + if [ $? -eq 0 ]; then + echo "Posted: $ref" + else + echo "FAILED: $ref" + fi + content="" + else + content="`echo "$content"; echo "$line"`" + fi + done + +} + +while [ 0 ]; do + echo "`date` -- SlightlyCyberpunk" + sap "https://www.twitter.com/SlightlyCyberpunkDoesntActuallyHaveTwitter" "MastodonToken" & + pid=$! + i=0 + while [ `ps -p $pid | wc -l` -gt 1 ]; do + sleep 2 + i=`expr $i + 1` + if [ $i -gt 30 ]; then + echo "`date` Killing sap" + kill $pid + fi + done + echo "`date` -- Sleeping...." + sleep 3530 + echo "`date` -- COFFEE TIME!" +done diff --git a/waitfor.js b/waitfor.js new file mode 100644 index 0000000..607e0d7 --- /dev/null +++ b/waitfor.js @@ -0,0 +1,292 @@ + +/* Loosely based on the waitfor.js sample from phantomjs */ +"use strict"; + +var system = require('system'); +var twiturl = system.args[1]; + +function getTweets() { + try { + console.log( + page.evaluate(function() { + var spanstrs = ""; + var articles = document.getElementsByTagName("article"); + for(var i = 0; i < articles.length; i++) { + var spanstr = ""; + + // Remove the buttons + var divs = articles[i].getElementsByTagName("div"); + var buttondiv = null; + for(var j = 0; j < divs.length; j++) { + if( divs[j].getAttribute("data-testid") == "like" ) { + buttondiv = divs[j].parentElement.parentElement; + } + } + if(buttondiv != null) { + buttondiv.parentElement.removeChild(buttondiv); + } + + // Remove the top link and grab the reference + var time = articles[i].getElementsByTagName("time")[0]; + // But first ensure it was posted within the past hour + if(!time.innerText.includes("m")) + continue; + var refelem = time.parentElement.parentElement.getElementsByTagName("a")[0]; + spanstr += "REF: "+time.parentElement.href+"\n"; + time = time.parentElement.parentElement; + time.parentElement.removeChild(time); + + // Add images + imgs = articles[i].getElementsByTagName("img"); + for(j = 0; j < imgs.length; j++) { + if(imgs[j].parentElement.getAttribute("data-testid") == "tweetPhoto") { + spanstr += "IMAGE: "+imgs[j].src+"\n"; + } + } + + // Add videos + vids = articles[i].getElementsByTagName("video"); + for(j = 0; j < vids.length; j++) { + spanstr += "VIDEO: "+vids[j].src+"\n"; + } + + // Search for links + var divs = articles[i].getElementsByTagName("div"); + var linkid = null; + for(var j = 0; j < divs.length; j++) { + if( divs[j].getAttribute("data-testid") == "card.wrapper" ) { + linkid = divs[j].getElementsByTagName("a")[0]; + spanstr += "LINK: "+linkid.href+"|"+linkid.innerText+"\n"; + break; + } + } + // Remove the link preview block + if(linkid != null) { + linkid = linkid.parentElement.parentElement; + linkid.parentElement.removeChild(linkid); + } + + // Retweets + if(articles[i].innerText.includes("Quote Tweet") && + articles[i].getElementsByTagName("time").length > 0) { + var itime = articles[i].getElementsByTagName("time")[0].parentElement.parentElement; + itime.parentElement.removeChild(itime); + } + + // TODO: ?Retweets?, replies + // if it contains , add refelem.innerText after retweeted? + + // Add the text + var links = articles[i].getElementsByTagName("a"); + for(var j = 0; j < links.length; j++) { + links[j].parentElement.outerHTML = links[j].parentElement.innerText; + } + var divs = articles[i].getElementsByTagName("div"); + for(var j = 0; j < divs.length; j++) { + divs[j].outerHTML = divs[j].innerText.replace(/\n/g, "
"); + } + spanstr += articles[i].innerText. + replace(/\n@/g, " @"). + replace("Quote Tweet\n", "\nQuote Tweet: "). + replace("Quote Tweet: \n", "Quote Tweet: "). + replace(" Retweeted\n", " Retweeted: "+refelem.innerText+"\n"); + + + spanstrs += spanstr + "\n-------------------------\n"; + } + return spanstrs; + } + ) + ); + } catch(e) { + console.log("EXCEPTION"); + } +} + +function getTweets1() { +// var tu1 = twiturl; + try { + console.log( + page.evaluate(function() { + var spanstrs=""; //"LOC: "+document.location; +// console.log("LOC: "+document.location); + var divs = document.getElementsByTagName("div"); + + for(var i = 0; i < divs.length; i++) { +// console.log("I: "+i); + + if(divs[i].getAttribute("role") == "button") { + divs[i].parentElement.removeChild(divs[i]); + } + + // INDIVIDUAL TWEET + if(divs[i].getAttribute("data-testid") == "tweet") { + var spanstr = ""; + var statusref = ""; + + var socialContext = "\n"; + // Check if this is a retweet + var siblingSpans = divs[i].previousElementSibling.getElementsByTagName("span"); + for(var j = 0; j < siblingSpans.length; j++) { + if(siblingSpans[j].getAttribute("data-testId") == "socialContext") { + //socialContext = siblingSpans[j].innerText+"\n\n"; + // Set the tweet reference + /*links = divs[i].getElementsByTagName("a"); + for(k = 0; k < links.length; k++) { + if(links[k].getAttribute("role") == "link") { + statusref = links[k].href; + spanstr += "aREF: "+statusref+"\n"; + } + }*/ + var timeElem = divs[i].getElementsByTagName('time')[0].parentElement; + statusref = timeElem.href; + spanstr += "REF: "+statusref+"\n"; + socialContext = "Retweet from: "+timeElem.parentElement.getElementsByTagName("span")[2].innerText+"\n"; + break; + } + } + // Check if this is a reply + var links = divs[i].getElementsByTagName("a"); + for(var j = 0; j < links.length; j++) { +/* if(links[j].parentElement.parentElement.parentElement.parentElement.parentElement.parentElement == divs[i]) + spanstr += "|"+links[j].parentElement.parentElement.innerText.replace("\n\n@", " @");*/ + if(links[j].parentElement.parentElement.innerText.substring(0,11) == "Replying to") { + socialContext = links[j].parentElement.parentElement.innerText.replace("\n", " "); + spanstr+="\nOK\n"; +// links[j].parentElement.parentElement.parentElement.removeChild(links[j].parentElement.parentElement); +// spanstr += "(("+divs[i].children[1].children[1].children[0].innerText+"))"; + } else { + spanstr += "**"+links[j].parentElement.parentElement.innerText+"**"; + } + } + spanstr += socialContext; + + // Get timestamp + var time = divs[i].getElementsByTagName("time")[0].innerText; +/* + if( !time.includes("m") ) { + continue; + }*/ + + // spanstr += "\n!links!\n"; + // Get any links + links = divs[i].getElementsByTagName("a"); + for(j = 0; j < links.length; j++) { + if(links[j].href.toLowerCase().includes(document.location.href.toLowerCase()) && + links[j].href.toLowerCase() != document.location.href.toLowerCase()) { + statusref = links[j].href; + spanstr += "REF: "+statusref+"\n"; + links[j].parentElement.removeChild(links[j]); + } else if(links[j].innerText[0] == "#" || + links[j].innerText[0] == "@") { + links[j].outerHTML = ""+links[j].innerText+""; +// } else if(links[j].parentElement.parentElement.getAttributesByTagName("time").length > 0) { +// links[j].parentElement.removeChild(links[j]); + } else if(links[j].href.toLowerCase() != document.location.href.toLowerCase()) { + spanstr += "LINK: "+links[j].href+"|"+links[j].innerText+"\n"; + //links[j].parentElement.removeChild(links[j]); + } else { + links[j].parentElement.removeChild(links[j]); + } + } + + // Get any video + vids = divs[i].getElementsByTagName("video"); + for(j = 0; j < vids.length; j++) { + spanstr += "VIDEO: "+vids[j].src+"\n"; + } + + // Get any images + imgs = divs[i].getElementsByTagName("img"); + for(j = 0; j < imgs.length; j++) { + if(imgs[j].parentElement.getAttribute("data-testid") == "tweetPhoto") { + spanstr += "IMAGE: "+imgs[j].src+"\n"; + } + } + + //spanstr += "\n!text!\n"; + // Get any text + spans = divs[i].getElementsByTagName("span"); + var quote=false; + for(j = 3; j < spans.length; j++) { + if(spans[j].innerText == "http://" || + spans[j].innerText == "https://" ) { + spanstr += spans[j].parentElement.href; + } else if(spans[j].innerText = "Quote Tweet") { + quote=true; + } else if(quote==true && spans.length > j+1 && + "@"+spans[j].innerText == spans[j+1].innerText) { + quote=false; + } else if( spans[j].innerText.length > 3 && (spans[j][0] != "@" || spans[j] != spans[j-1])) { + spanstr += spans[j].innerText+""; + } + } + if( spanstr.length > 1) { + spanstrs += spanstr + "\n-------------------------\n"; + } + } + // END INDIVIDUAL TWEET + } + return spanstrs; + })); + } catch(e) { + return "EXCEPTION"; + } +} + +function waitFor(testFx, onReady, timeOutMillis) { + var maxtimeOutMillis = timeOutMillis ? timeOutMillis : 10000, //< Default Max Timout is 10s + start = new Date().getTime(), + condition = false, + interval = setInterval(function() { + if ( (new Date().getTime() - start < maxtimeOutMillis) && !condition ) { + // If not time-out yet and condition not yet fulfilled + condition = (typeof(testFx) === "string" ? eval(testFx) : testFx()); //< defensive code + } else { + if(!condition) { + // If condition still not fulfilled (timeout but condition is 'false') + console.log("'waitFor()' timeout"); + console.log("CHECK3"); + getTweets(); + console.log("CHECK4"); + phantom.exit(); + } else { + // Condition fulfilled (timeout and/or condition is 'true') + console.log("'waitFor()' finished in " + (new Date().getTime() - start) + "ms."); + typeof(onReady) === "string" ? eval(onReady) : onReady(); //< Do what it's supposed to do once the condition is fulfilled + clearInterval(interval); //< Stop this interval + } + } + }, 250); //< repeat check every 250ms +}; + + +var page = require('webpage').create(); +page.viewportSize = {width: 1280, height: 1080}; + +// Open Twitter on 'sencha' profile and, onPageLoad, do... +page.open(twiturl, function (status) { + // Check for page load success + if (status !== "success") { + console.log("Unable to access network"); + } else { + // Wait for 'signin-dropdown' to be visible + waitFor(function() { + // Check in the page if a specific element is now visible + return page.evaluate(function() { + try { + return document.getElementsByTagName("div").length > 1400; + } catch(e) { + return false; + } + }); + }, function() { +// page.render('example.png') +// console.log("The sign-in dialog should be visible now."); + console.log("CHECK1"); + getTweets(); + console.log("CHECK2"); + phantom.exit(); + }); + } +});