Up until now, I was working on the list of Tumblir blogs by type of the writter. Now I wanted to get the text of their blog posts.
What I made sure here to no longer bother with the timeout because of the API restrictions, since the daily restrictions were always reached quite soon. And it did not matter if I run the script6 b6y hand, or if I had the time our function programmed in.
import pytumblr import re import os import sys #this is a regex, to be able to get the url of the Tumblr blog from the url post tumblr_url = r"\w+.tumblr.com" blog_url = r"/(\d+)/([\w-]+)" name_after_url = r"/\d+/[\w-]+" digits_only = r"/(\d+)" offset = 0 webpages = get_all_webpages("file.csv") write_set_to_file(webpages, "file.csv") url = list(webpages) tag = "" # Authenticate via OAuth client = pytumblr.TumblrRestClient() def get_all_webpages(filename): allwebpages = set() with open(filename, "r") as read: content = read.readlines() for line in content: line = line.split("\t") allwebpages.add(line.strip()) return allwebpages def write_set_to_file(settowrite, singlefile): with open(singlefile, "w") as write: for element in settowrite: write.write(element + "\n") return None def get_blog_post(mbti, url, offset): blogpost = client.posts(url, type='text', limit=20, offset=offset) try: if not blogpost["posts"]: print("Finished with blog: " + url) return "finished" url = blogpost["posts"]["post_url"] title = blogpost["posts"]["title"] body = blogpost["posts"]["body"] except: print(blogpost) if blogpost["meta"]["msg"] == "Limit Exceeded": print("Next offset for " + mbti + " is: " + str(offset)) with open("offset", "w") as write: write.write(str(offset)) return None try: os.mkdir(mbti) except: pass print(url) if re.compile(name_after_url).search(url) is not None: blog_name = re.search(blog_url, url.strip()) blog_name = blog_name.groups() blog_name = "-".join(blog_name) else: blog_name = re.search(digits_only, url.strip()) blog_name = blog_name.groups() blog_name = "".join(blog_name) with open(mbti + "/" + blog_name + ".txt", "w") as write: if not title: title = "" if not body: body = "" title = title.encode("utf8") body = body.encode("utf8") write.write(title + "\n\n\n" + body) return offset + 20 while 1: offset = get_blog_post(tag, url, offset) if offset == None: write_set_to_file(webpages, "blogsistp-cleaned-2.csv") sys.exit() if offset == "finished": webpages.remove(url) if not webpages: break url = list(webpages) offset = 0 write_set_to_file(webpages, "file-out.csv")
Because of this restriction, I decided to abandon this way of getting information, since it would take at least a month to get the amount of data that I wanted.
Since I planed to use this for the school presentation as well, that was not something that I could afford. So I decided to try and find a different source of data.