mirror of
https://github.com/developersIndia/deviras.git
synced 2024-12-22 16:19:50 +05:30
remove reading reddit collection for community threads collection
This commit is contained in:
parent
e41335edc3
commit
4f9f286e82
@ -1,9 +1,11 @@
|
|||||||
name : Community Threads Wiki Updater
|
name : Community Threads Wiki Updater
|
||||||
|
|
||||||
on:
|
on:
|
||||||
schedule:
|
|
||||||
- cron: '0 0 * * *' # This cron expression triggers the workflow every day at midnight UTC
|
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
post_url:
|
||||||
|
description: 'The URL of the Reddit post to add'
|
||||||
|
required: true
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
@ -32,4 +34,4 @@ jobs:
|
|||||||
GIST_TOKEN: ${{ secrets.GIST_TOKEN }}
|
GIST_TOKEN: ${{ secrets.GIST_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
cd community-threads
|
cd community-threads
|
||||||
python main.py
|
python main.py ${{ github.event.inputs.post_url }}
|
@ -1,5 +1,6 @@
|
|||||||
import praw
|
import praw
|
||||||
import os
|
import os
|
||||||
|
import argparse
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import json
|
import json
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
@ -36,13 +37,26 @@ def update_gist(gist_id, filename, content, description=""):
|
|||||||
)
|
)
|
||||||
return response.json()
|
return response.json()
|
||||||
|
|
||||||
|
# farewell, reddit collections
|
||||||
|
# def get_collection(reddit):
|
||||||
|
# collection = reddit.subreddit(sub).collections(
|
||||||
|
# permalink="https://reddit.com/r/developersIndia/collection/958aef35-f9cb-414d-ab33-08bc639e47de"
|
||||||
|
# )
|
||||||
|
# return collection
|
||||||
|
|
||||||
def get_collection(reddit):
|
def get_post_data(reddit, post_url):
|
||||||
collection = reddit.subreddit(sub).collections(
|
submission = reddit.submission(url=post_url)
|
||||||
permalink="https://reddit.com/r/developersIndia/collection/958aef35-f9cb-414d-ab33-08bc639e47de"
|
post = {
|
||||||
)
|
"title": submission.title,
|
||||||
return collection
|
"url": submission.url,
|
||||||
|
"id": submission.id,
|
||||||
|
"num_comments": submission.num_comments,
|
||||||
|
"created_at": datetime.utcfromtimestamp(
|
||||||
|
submission.created_utc
|
||||||
|
).isoformat(),
|
||||||
|
"flair_text": submission.link_flair_text,
|
||||||
|
}
|
||||||
|
return post
|
||||||
|
|
||||||
def update_wiki(reddit, wikipage, posts):
|
def update_wiki(reddit, wikipage, posts):
|
||||||
# Group posts by year
|
# Group posts by year
|
||||||
@ -78,6 +92,10 @@ def update_wiki(reddit, wikipage, posts):
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description='Update Community Threads Collection.')
|
||||||
|
parser.add_argument('post_url', help='The URL of the Reddit post to add.')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
reddit = praw.Reddit(
|
reddit = praw.Reddit(
|
||||||
client_id=client_id,
|
client_id=client_id,
|
||||||
client_secret=client_secret,
|
client_secret=client_secret,
|
||||||
@ -86,38 +104,13 @@ def main():
|
|||||||
user_agent=f"Automod reader by u/{username}",
|
user_agent=f"Automod reader by u/{username}",
|
||||||
)
|
)
|
||||||
|
|
||||||
collection = get_collection(reddit)
|
|
||||||
|
|
||||||
saved_collection_posts = json.loads(get_gist_content(gist_id))
|
saved_collection_posts = json.loads(get_gist_content(gist_id))
|
||||||
saved_collection_ids = [post["id"] for post in saved_collection_posts["posts"]]
|
saved_collection_ids = [post["id"] for post in saved_collection_posts["posts"]]
|
||||||
|
|
||||||
print(f"Database was last updated on {saved_collection_posts['collection_last_updated']}")
|
print(f"Database was last updated on {saved_collection_posts['collection_last_updated']}")
|
||||||
print(f"Collection was last updated on {datetime.utcfromtimestamp(collection.last_update_utc).isoformat()}")
|
|
||||||
|
|
||||||
if (
|
|
||||||
saved_collection_posts["collection_last_updated"]
|
|
||||||
!= datetime.utcfromtimestamp(collection.last_update_utc).isoformat()
|
|
||||||
):
|
|
||||||
print("Collection was updated, getting new posts data...")
|
|
||||||
|
|
||||||
# given 2 lists find non-common elements
|
|
||||||
db_posts = set(saved_collection_ids)
|
|
||||||
collection_posts = []
|
|
||||||
for submission in collection:
|
|
||||||
collection_posts.append(submission.id)
|
|
||||||
collection_posts = set(collection_posts)
|
|
||||||
|
|
||||||
new_posts = list(collection_posts - db_posts)
|
|
||||||
deleted_posts = list(db_posts - collection_posts)
|
|
||||||
|
|
||||||
print(f"Found {len(new_posts)} new posts!")
|
|
||||||
print(f"Found {len(deleted_posts)} deleted posts!")
|
|
||||||
|
|
||||||
posts = []
|
posts = []
|
||||||
# load the saved collection posts data
|
|
||||||
for submission_id in saved_collection_posts["posts"]:
|
for submission_id in saved_collection_posts["posts"]:
|
||||||
if submission_id["id"] in deleted_posts:
|
|
||||||
continue
|
|
||||||
post = {
|
post = {
|
||||||
"title": submission_id["title"],
|
"title": submission_id["title"],
|
||||||
"url": submission_id["url"],
|
"url": submission_id["url"],
|
||||||
@ -128,28 +121,13 @@ def main():
|
|||||||
}
|
}
|
||||||
posts.append(post)
|
posts.append(post)
|
||||||
|
|
||||||
# get the new posts data
|
new_post = get_post_data(reddit, args.post_url)
|
||||||
for submission_id in new_posts:
|
if new_post["id"] not in saved_collection_ids:
|
||||||
submission = reddit.submission(submission_id)
|
posts.append(new_post)
|
||||||
post = {
|
|
||||||
"title": submission.title,
|
|
||||||
"url": submission.url,
|
|
||||||
"id": submission.id,
|
|
||||||
"num_comments": submission.num_comments,
|
|
||||||
"created_at": datetime.utcfromtimestamp(
|
|
||||||
submission.created_utc
|
|
||||||
).isoformat(),
|
|
||||||
"flair_text": submission.link_flair_text,
|
|
||||||
}
|
|
||||||
posts.append(post)
|
|
||||||
|
|
||||||
# sort the posts by created_at
|
|
||||||
posts = sorted(posts, key=lambda k: k["created_at"])
|
posts = sorted(posts, key=lambda k: k["created_at"])
|
||||||
|
|
||||||
collection_json = {
|
collection_json = {
|
||||||
"collection_last_updated": datetime.utcfromtimestamp(
|
"collection_last_updated": datetime.utcnow().isoformat(),
|
||||||
collection.last_update_utc
|
|
||||||
).isoformat(),
|
|
||||||
"posts": posts,
|
"posts": posts,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -157,7 +135,7 @@ def main():
|
|||||||
print("Internal database updated successfully!")
|
print("Internal database updated successfully!")
|
||||||
update_wiki(reddit, "community-threads", posts)
|
update_wiki(reddit, "community-threads", posts)
|
||||||
else:
|
else:
|
||||||
print("Wiki is up to date!")
|
print("Post is already in the collection. No changes were made.")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
Loading…
Reference in New Issue
Block a user