--- /dev/null
+#!/bin/bash
+
+# MESSAGES:
+# SUBSCRIBE [frequency] [zip] [time] -- SUBSCRIBE daily 02895 7:00
+# UNSUBSCRIBE
+
+# TODO:
+# Remove hardcoded URLs and user info, put variables somewhere common
+# Maybe rewrite this to not be a shell script?
+
+# Fill in the app ID from openweathermap
+APPID=""
+
+# Fill in the token below from your Mastodon developer UI!
+setsubs() {
+token=""
+maxid="`cat weather.maxid`"
+
+curl -X GET \
+ -H "Authorization: Bearer ${token}" \
+ https://mastodon.slightlycyberpunk.com/api/v1/conversations?since_id=$maxid |\
+ sed 's/}},{"id"/}}\n{"id"/g' |\
+ grep -v '"account":{"id":"161","username":"forecast"' |\
+ sed 's/[{}]//g' |\
+ awk -v FS=',' -v OFS="|" '{
+ for(i=1;i<=NF;i++){
+ split($i,a,":");
+ if(a[1] == "\"content\"") {
+ content=$i;
+ } else if(a[1] == "\"username\"" && a[2] != "\"forecast\"") {
+ user=a[2];
+ } else if(a[1] == "\"last_status\"") {
+ id=a[3];
+ }
+ }}
+ END { print user,content,id }' |\
+ sort -u | grep -v '"username":"forecast"' |\
+ while read line; do
+ if [ -z "$line" -o "$line" == "||" ]; then
+ continue
+ fi
+ user="`echo "$line" | cut -d'|' -f1 | cut -d'"' -f2`"
+ id="`echo "$line" | cut -d'|' -f3 | cut -d'"' -f2`"
+ subs="`echo "$line" | cut -d'|' -f2 | grep "[^a-zA-Z]SUBSCRIBE" | sed 's/.*\([^A-Za-z]SUBSCRIBE [a-zA-Z]* [0-9]* [0-9]*:[0-9]*\).*/\1/g'`"
+ unsubs="`echo "$line" | cut -d'|' -f2 | grep "UNSUBSCRIBE"`"
+ greetings="`echo "$line" | cut -d'|' -f2 | grep "Hello"`"
+
+ if [ "$id" -gt "$maxid" ]; then
+ echo "$id" > weather.maxid
+ fi
+
+ if [ ! -z "$unsubs" ]; then
+ echo "UNSUBSCRIBING $user ($unsubs)"
+ curl -X POST \
+ -F 'Idempotency-Key=abc' \
+ -H "Authorization: Bearer ${token}" \
+ -F "status=ALL SUBSCRIPTIONS CANCELLED @${user}. If this doesn't work, please contact @admin@mastodon.slightlycyberpunk.com" \
+ -F 'visibility=direct' \
+ https://mastodon.slightlycyberpunk.com/api/v1/statuses
+ cat weather.subs | grep -v "^$user|" > weather.subs.2
+ mv weather.subs.2 weather.subs
+ fi
+
+ if [ "`cat weather.subs | grep "$user|$subs" | wc -l`" -eq 0 -a ! -z "$subs" ]; then
+ echo "SUBSCRIBING $user"
+ echo "$user|$subs" >> weather.subs
+
+ curl -X POST \
+ -F 'Idempotency-Key=abc' \
+ -H "Authorization: Bearer ${token}" \
+ -F "status=SUBSCRIPTION CONFIRMED @$user" \
+ -F 'visibility=direct' \
+ https://mastodon.slightlycyberpunk.com/api/v1/statuses
+ fi
+
+ if [ ! -z "$greetings" ]; then
+ echo "GEETING $user"
+
+ curl -X POST \
+ -F 'Idempotency-Key=abc' \
+ -H "Authorization: Bearer ${token}" \
+ -F "status=Greetings, @${user}! To subscribe for daily forecasts, send as a direct message:
+SUBSCRIBE daily [zipcode] [time]
+
+Currently only sending daily forecasts
+Zipcode must be a 5-digit US zipcode
+Time should be the time to send the forecast, in 24-hour EST format, as an even hour (ie, must end in :00)
+
+To unsubscribe from all forecasts, send as a direct message:
+UNSUBSCRIBE" \
+ -F 'visibility=direct' \
+ https://mastodon.slightlycyberpunk.com/api/v1/statuses 2>/dev/null
+ fi
+
+ done
+}
+
+while [ 0 ]; do
+ setsubs
+ cat weather.subs | sed 's/[ ]*|[ ]*/|/g' | grep "|SUBSCRIBE" | while read line; do
+ user="`echo "$line" | cut -d'|' -f1`"
+ type="`echo "$line" | cut -d'|' -f2 | cut -d' ' -f2`"
+ zip="`echo "$line" | cut -d'|' -f2 | cut -d' ' -f3`"
+ arg="`echo "$line" | cut -d'|' -f2 | cut -d' ' -f4`"
+
+ if [ "$type" == "daily" -a "$arg" == "`date +%H:00`" ]; then
+ LOCATION="$zip"
+ forecast="`curl --connect-timeout 20 \
+ "http://api.openweathermap.org/data/2.5/forecast/daily?APPID=$APPID&q=$LOCATION&mode=xml&units=imperial&cnt=1" 2>/dev/null |\
+ sed 's/></>\n</g' | egrep "<precipitation|<wind|<temperature" | awk -v FS='"' '
+/precipitation/ { if($2!=0){printf("%d chance of %s",$2,$6);}}
+/temperature/{if($14 == "fahrenheit"){unit="F";}else{unit="C";} printf("High of %d%s, Low of %d%s\n", $6, unit, $4, unit);
+ printf("%d%s this morning, to %d%s today, with %d%s this evining and %d%s tonight\n", $12,unit,$2,unit,$10,unit,$8,unit);}
+/windSpeed/ { windspeed=$6 }
+/windDirection/ { winddirection=$6 }
+END{ printf("And a %s from the %s\n", windspeed, winddirection);}'| cut -c1-460`"
+
+ curl -X POST \
+ -H "Authorization: Bearer ${token}" \
+ -F "status=Today's Forecast for @$user:
+${forecast}" \
+ -F "visibility=direct" \
+ https://mastodon.slightlycyberpunk.com/api/v1/statuses
+ fi
+ done
+ sleep 3500
+done
--- /dev/null
+# TODO:
+# Refresh tokens automatically
+# Mover user/token info
+# Remove debugging stuff
+# Basically refactor everything
+
+
+# Scrape and post
+sap() {
+ url="$1"
+ token="$2"
+ phantomjs waitfor.js "${url}" 2>/dev/null | uniq | grep -v waitFor | grep -v TypeError | uniq | while read line; do
+ if [ ! -z "`echo "$line" | grep "^CHECK"`" ]; then
+ continue
+ fi
+ if [ "$line" == "-------------------------" ]; then
+ if [ -z "`echo "$content" | grep -v '^REF'`" ]; then
+ continue
+ fi
+
+ # Extract source reference
+ ref="`echo "$content" | grep '^REF' | tail -1 | cut -d':' -f2-`"
+ refline="Original post on Twitter: ${ref}"
+
+ # Extract video links
+ vid="`echo "$content" | grep '^VIDEO:' | tail -1 | cut -d':' -f2- | sed 's/^ *//g'`"
+ media_ids=""
+ if [ ! -z "$vid" ]; then
+ echo "Pulling video..."
+ ffmpeg -i "$vid" -c copy -bsf:a aac_adtstoasc output.mp4 >/dev/null 2>&1
+ echo "Downloaded."
+ curl -X POST \
+ -H "Authorization: Bearer ${token}" \
+ -F 'file=@output.mp4' \
+ https://mastodon.slightlycyberpunk.com/api/v1/media > vjson
+ mediaid="`cat vjson | sed 's/.*"id":"\([0-9]*\)".*/\1/g'`"
+ media_ids="media_ids[]=$mediaid"
+ echo "IDs: $media_ids"
+ fi
+
+ while read img; do
+ if [ ! -z "$img" ]; then
+ format="`echo "$img" | sed 's/.*format=\([a-zA-Z0-9]*\)&.*/\1/g'`"
+ echo "Pulling image.${format}..."
+ wget -O "image.${format}" "$img" #>/dev/null 2>&1
+ curl -X POST \
+ -H "Authorization: Bearer ${token}" \
+ -F "file=@image.${format}" \
+ https://mastodon.slightlycyberpunk.com/api/v1/media > vjson
+ mediaid="`cat vjson | sed 's/.*"id":"\([0-9]*\)".*/\1/g'`"
+ if [ ! -z "$media_ids" ]; then
+ media_ids="$media_ids&media_ids[]=$mediaid"
+ else
+ media_ids="media_ids[]=$mediaid"
+ fi
+ echo "IDs: $media_ids"
+ fi
+ done < <(echo "$content" | grep "^IMAGE:" | cut -d':' -f2- | sed 's/^ *//g')
+
+ echo "IDs: $media_ids"
+
+ while read link; do
+ content="${content}
+
+${link}"
+ done < <(echo "$content" | grep "^LINK:" | cut -d':' -f2- | sed 's/^ *//g' | cut -d'|' -f1 | cut -d'?' -f1)
+
+echo "CONTENT: $content"
+echo "REFLINE: $refline"
+
+ fullcontent="`echo "$content" | grep -v '^REF' | grep -v '^VIDEO' | grep -v '^IMAGE:' | grep -v '^LINK:' | cut -c1-430 | sed 's/"/\\\"/g'`
+
+$refline"
+
+echo "POSTING:"
+echo "${fullcontent}"
+
+ if [ ! -z "$media_ids" ]; then
+ curl -X POST \
+ -F 'Idempotency-Key=abc' \
+ -H "Authorization: Bearer ${token}" \
+ -F "status=\"${fullcontent}\"" \
+ -F "$media_ids" \
+ -F 'visibility=private' \
+ https://mastodon.slightlycyberpunk.com/api/v1/statuses
+
+ if [ -f output.mp4 ]; then
+ rm output.mp4
+ elif [ -f image.* ]; then
+ rm image.*
+ fi
+ else
+ curl -X POST \
+ -F 'Idempotency-Key=abc' \
+ -H "Authorization: Bearer ${token}" \
+ -F "status=\"${fullcontent}\"" \
+ -F 'visibility=private' \
+ https://mastodon.slightlycyberpunk.com/api/v1/statuses
+ fi
+
+ if [ $? -eq 0 ]; then
+ echo "Posted: $ref"
+ else
+ echo "FAILED: $ref"
+ fi
+ content=""
+ else
+ content="`echo "$content"; echo "$line"`"
+ fi
+ done
+
+}
+
+while [ 0 ]; do
+ echo "`date` -- SlightlyCyberpunk"
+ sap "https://www.twitter.com/SlightlyCyberpunkDoesntActuallyHaveTwitter" "MastodonToken" &
+ pid=$!
+ i=0
+ while [ `ps -p $pid | wc -l` -gt 1 ]; do
+ sleep 2
+ i=`expr $i + 1`
+ if [ $i -gt 30 ]; then
+ echo "`date` Killing sap"
+ kill $pid
+ fi
+ done
+ echo "`date` -- Sleeping...."
+ sleep 3530
+ echo "`date` -- COFFEE TIME!"
+done
--- /dev/null
+
+/* Loosely based on the waitfor.js sample from phantomjs */
+"use strict";
+
+var system = require('system');
+var twiturl = system.args[1];
+
+function getTweets() {
+ try {
+ console.log(
+ page.evaluate(function() {
+ var spanstrs = "";
+ var articles = document.getElementsByTagName("article");
+ for(var i = 0; i < articles.length; i++) {
+ var spanstr = "";
+
+ // Remove the buttons
+ var divs = articles[i].getElementsByTagName("div");
+ var buttondiv = null;
+ for(var j = 0; j < divs.length; j++) {
+ if( divs[j].getAttribute("data-testid") == "like" ) {
+ buttondiv = divs[j].parentElement.parentElement;
+ }
+ }
+ if(buttondiv != null) {
+ buttondiv.parentElement.removeChild(buttondiv);
+ }
+
+ // Remove the top link and grab the reference
+ var time = articles[i].getElementsByTagName("time")[0];
+ // But first ensure it was posted within the past hour
+ if(!time.innerText.includes("m"))
+ continue;
+ var refelem = time.parentElement.parentElement.getElementsByTagName("a")[0];
+ spanstr += "REF: "+time.parentElement.href+"\n";
+ time = time.parentElement.parentElement;
+ time.parentElement.removeChild(time);
+
+ // Add images
+ imgs = articles[i].getElementsByTagName("img");
+ for(j = 0; j < imgs.length; j++) {
+ if(imgs[j].parentElement.getAttribute("data-testid") == "tweetPhoto") {
+ spanstr += "IMAGE: "+imgs[j].src+"\n";
+ }
+ }
+
+ // Add videos
+ vids = articles[i].getElementsByTagName("video");
+ for(j = 0; j < vids.length; j++) {
+ spanstr += "VIDEO: "+vids[j].src+"\n";
+ }
+
+ // Search for links
+ var divs = articles[i].getElementsByTagName("div");
+ var linkid = null;
+ for(var j = 0; j < divs.length; j++) {
+ if( divs[j].getAttribute("data-testid") == "card.wrapper" ) {
+ linkid = divs[j].getElementsByTagName("a")[0];
+ spanstr += "LINK: "+linkid.href+"|"+linkid.innerText+"\n";
+ break;
+ }
+ }
+ // Remove the link preview block
+ if(linkid != null) {
+ linkid = linkid.parentElement.parentElement;
+ linkid.parentElement.removeChild(linkid);
+ }
+
+ // Retweets
+ if(articles[i].innerText.includes("Quote Tweet") &&
+ articles[i].getElementsByTagName("time").length > 0) {
+ var itime = articles[i].getElementsByTagName("time")[0].parentElement.parentElement;
+ itime.parentElement.removeChild(itime);
+ }
+
+ // TODO: ?Retweets?, replies
+ // if it contains <retweeted>, add refelem.innerText after retweeted?
+
+ // Add the text
+ var links = articles[i].getElementsByTagName("a");
+ for(var j = 0; j < links.length; j++) {
+ links[j].parentElement.outerHTML = links[j].parentElement.innerText;
+ }
+ var divs = articles[i].getElementsByTagName("div");
+ for(var j = 0; j < divs.length; j++) {
+ divs[j].outerHTML = divs[j].innerText.replace(/\n/g, "<br/>");
+ }
+ spanstr += articles[i].innerText.
+ replace(/\n@/g, " @").
+ replace("Quote Tweet\n", "\nQuote Tweet: ").
+ replace("Quote Tweet: \n", "Quote Tweet: ").
+ replace(" Retweeted\n", " Retweeted: "+refelem.innerText+"\n");
+
+
+ spanstrs += spanstr + "\n-------------------------\n";
+ }
+ return spanstrs;
+ }
+ )
+ );
+ } catch(e) {
+ console.log("EXCEPTION");
+ }
+}
+
+function getTweets1() {
+// var tu1 = twiturl;
+ try {
+ console.log(
+ page.evaluate(function() {
+ var spanstrs=""; //"LOC: "+document.location;
+// console.log("LOC: "+document.location);
+ var divs = document.getElementsByTagName("div");
+
+ for(var i = 0; i < divs.length; i++) {
+// console.log("I: "+i);
+
+ if(divs[i].getAttribute("role") == "button") {
+ divs[i].parentElement.removeChild(divs[i]);
+ }
+
+ // INDIVIDUAL TWEET
+ if(divs[i].getAttribute("data-testid") == "tweet") {
+ var spanstr = "";
+ var statusref = "";
+
+ var socialContext = "\n";
+ // Check if this is a retweet
+ var siblingSpans = divs[i].previousElementSibling.getElementsByTagName("span");
+ for(var j = 0; j < siblingSpans.length; j++) {
+ if(siblingSpans[j].getAttribute("data-testId") == "socialContext") {
+ //socialContext = siblingSpans[j].innerText+"\n\n";
+ // Set the tweet reference
+ /*links = divs[i].getElementsByTagName("a");
+ for(k = 0; k < links.length; k++) {
+ if(links[k].getAttribute("role") == "link") {
+ statusref = links[k].href;
+ spanstr += "aREF: "+statusref+"\n";
+ }
+ }*/
+ var timeElem = divs[i].getElementsByTagName('time')[0].parentElement;
+ statusref = timeElem.href;
+ spanstr += "REF: "+statusref+"\n";
+ socialContext = "Retweet from: "+timeElem.parentElement.getElementsByTagName("span")[2].innerText+"\n";
+ break;
+ }
+ }
+ // Check if this is a reply
+ var links = divs[i].getElementsByTagName("a");
+ for(var j = 0; j < links.length; j++) {
+/* if(links[j].parentElement.parentElement.parentElement.parentElement.parentElement.parentElement == divs[i])
+ spanstr += "|"+links[j].parentElement.parentElement.innerText.replace("\n\n@", " @");*/
+ if(links[j].parentElement.parentElement.innerText.substring(0,11) == "Replying to") {
+ socialContext = links[j].parentElement.parentElement.innerText.replace("\n", " ");
+ spanstr+="\nOK\n";
+// links[j].parentElement.parentElement.parentElement.removeChild(links[j].parentElement.parentElement);
+// spanstr += "(("+divs[i].children[1].children[1].children[0].innerText+"))";
+ } else {
+ spanstr += "**"+links[j].parentElement.parentElement.innerText+"**";
+ }
+ }
+ spanstr += socialContext;
+
+ // Get timestamp
+ var time = divs[i].getElementsByTagName("time")[0].innerText;
+/*
+ if( !time.includes("m") ) {
+ continue;
+ }*/
+
+ // spanstr += "\n!links!\n";
+ // Get any links
+ links = divs[i].getElementsByTagName("a");
+ for(j = 0; j < links.length; j++) {
+ if(links[j].href.toLowerCase().includes(document.location.href.toLowerCase()) &&
+ links[j].href.toLowerCase() != document.location.href.toLowerCase()) {
+ statusref = links[j].href;
+ spanstr += "REF: "+statusref+"\n";
+ links[j].parentElement.removeChild(links[j]);
+ } else if(links[j].innerText[0] == "#" ||
+ links[j].innerText[0] == "@") {
+ links[j].outerHTML = "<span>"+links[j].innerText+"</span>";
+// } else if(links[j].parentElement.parentElement.getAttributesByTagName("time").length > 0) {
+// links[j].parentElement.removeChild(links[j]);
+ } else if(links[j].href.toLowerCase() != document.location.href.toLowerCase()) {
+ spanstr += "LINK: "+links[j].href+"|"+links[j].innerText+"\n";
+ //links[j].parentElement.removeChild(links[j]);
+ } else {
+ links[j].parentElement.removeChild(links[j]);
+ }
+ }
+
+ // Get any video
+ vids = divs[i].getElementsByTagName("video");
+ for(j = 0; j < vids.length; j++) {
+ spanstr += "VIDEO: "+vids[j].src+"\n";
+ }
+
+ // Get any images
+ imgs = divs[i].getElementsByTagName("img");
+ for(j = 0; j < imgs.length; j++) {
+ if(imgs[j].parentElement.getAttribute("data-testid") == "tweetPhoto") {
+ spanstr += "IMAGE: "+imgs[j].src+"\n";
+ }
+ }
+
+ //spanstr += "\n!text!\n";
+ // Get any text
+ spans = divs[i].getElementsByTagName("span");
+ var quote=false;
+ for(j = 3; j < spans.length; j++) {
+ if(spans[j].innerText == "http://" ||
+ spans[j].innerText == "https://" ) {
+ spanstr += spans[j].parentElement.href;
+ } else if(spans[j].innerText = "Quote Tweet") {
+ quote=true;
+ } else if(quote==true && spans.length > j+1 &&
+ "@"+spans[j].innerText == spans[j+1].innerText) {
+ quote=false;
+ } else if( spans[j].innerText.length > 3 && (spans[j][0] != "@" || spans[j] != spans[j-1])) {
+ spanstr += spans[j].innerText+"";
+ }
+ }
+ if( spanstr.length > 1) {
+ spanstrs += spanstr + "\n-------------------------\n";
+ }
+ }
+ // END INDIVIDUAL TWEET
+ }
+ return spanstrs;
+ }));
+ } catch(e) {
+ return "EXCEPTION";
+ }
+}
+
+function waitFor(testFx, onReady, timeOutMillis) {
+ var maxtimeOutMillis = timeOutMillis ? timeOutMillis : 10000, //< Default Max Timout is 10s
+ start = new Date().getTime(),
+ condition = false,
+ interval = setInterval(function() {
+ if ( (new Date().getTime() - start < maxtimeOutMillis) && !condition ) {
+ // If not time-out yet and condition not yet fulfilled
+ condition = (typeof(testFx) === "string" ? eval(testFx) : testFx()); //< defensive code
+ } else {
+ if(!condition) {
+ // If condition still not fulfilled (timeout but condition is 'false')
+ console.log("'waitFor()' timeout");
+ console.log("CHECK3");
+ getTweets();
+ console.log("CHECK4");
+ phantom.exit();
+ } else {
+ // Condition fulfilled (timeout and/or condition is 'true')
+ console.log("'waitFor()' finished in " + (new Date().getTime() - start) + "ms.");
+ typeof(onReady) === "string" ? eval(onReady) : onReady(); //< Do what it's supposed to do once the condition is fulfilled
+ clearInterval(interval); //< Stop this interval
+ }
+ }
+ }, 250); //< repeat check every 250ms
+};
+
+
+var page = require('webpage').create();
+page.viewportSize = {width: 1280, height: 1080};
+
+// Open Twitter on 'sencha' profile and, onPageLoad, do...
+page.open(twiturl, function (status) {
+ // Check for page load success
+ if (status !== "success") {
+ console.log("Unable to access network");
+ } else {
+ // Wait for 'signin-dropdown' to be visible
+ waitFor(function() {
+ // Check in the page if a specific element is now visible
+ return page.evaluate(function() {
+ try {
+ return document.getElementsByTagName("div").length > 1400;
+ } catch(e) {
+ return false;
+ }
+ });
+ }, function() {
+// page.render('example.png')
+// console.log("The sign-in dialog should be visible now.");
+ console.log("CHECK1");
+ getTweets();
+ console.log("CHECK2");
+ phantom.exit();
+ });
+ }
+});
git://git.slightlycyberpunk.com / mastodon-tools.git / commitdiff