Browse Source

Implementation of welcome bot (service)

merge-requests/1/head 20170709.1
KemoNine 4 years ago
parent
commit
a98d5cc27c
  1. 220
      BotStreamListeners.py
  2. 18
      README.md
  3. 58
      Toot.py
  4. 5
      configs/example_generic_config.yaml
  5. 1
      configs/example_rss.yaml
  6. 1
      configs/example_rss_custom_logic.yaml
  7. 16
      configs/example_welcome_bot.service
  8. 26
      configs/example_welcome_config.yaml
  9. 136
      toot_bot.py

220
BotStreamListeners.py

@ -0,0 +1,220 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Adjust PyLint settings
# pylint: disable=C0301
''' Different stream listener implementations used by bots '''
import sys, pprint
import sqlite3
import copy
from mastodon import Mastodon, StreamListener
from Toot import toot
class WelcomeBot(StreamListener):
''' Implementation of the Mastodon.py StreamListener class for welcome bot purposes '''
def __init__(self, bot_config):
StreamListener.__init__(self)
self.bot_config = bot_config
# Get access to cache
self.conn = sqlite3.connect(self.bot_config['welcome']['cache_file'])
self.cursor = self.conn.cursor()
# Ensure cache table has been created
self.cursor.execute('''create table if not exists welcome_cache (
username varchar(2048) primary key,
seen_timestamp timestamp
);'''
)
self.cursor.execute('''create table if not exists toot_cache (
toot_id integer primary key
);'''
)
self.conn.commit()
# Replay any toots that were missed while offline and welcome new users
self.replay_toots(True)
def __del__(self):
# Cleanup connection to sqlite database for welcome cache
self.conn.commit()
self.conn.close()
def fetch_remaining(self, mastodon, first_page):
''' Work around for odd behavior in Mastodon.py's official fetch_remaining code '''
# FIXME: Remove this method when below GitHub issue is closed and a new release available
# FIXME: Don't forget to update minimum version of Mastodon.py to match when the fix is released
# https://github.com/halcy/Mastodon.py/issues/59
first_page = copy.deepcopy(first_page)
all_pages = []
current_page = first_page
while current_page != None and current_page:
all_pages.extend(current_page)
current_page = mastodon.fetch_next(current_page)
return all_pages
def replay_toots(self, recurse=False):
''' Replay toots that were posted while the bot was offline '''
# Setup Mastodon API
mastodon = Mastodon(
client_id=self.bot_config['config']['client_cred_file'],
access_token=self.bot_config['config']['user_cred_file'],
api_base_url=self.bot_config['config']['api_base_url']
)
self.cursor.execute('select max(toot_id) from toot_cache;')
last_seen_toot_id = self.cursor.fetchone()[0]
if last_seen_toot_id is None:
last_seen_toot_id = 0
first_page = mastodon.timeline_local(since_id=last_seen_toot_id)
all_pages = self.fetch_remaining(mastodon, first_page)
# Catch up ALL welcome messages that may have been missed
for status in all_pages:
self.welcome_user(status)
# Update max seen toot id (any calls to welcome_user will move the max)
self.cursor.execute('select max(toot_id) from toot_cache;')
last_seen_toot_id = self.cursor.fetchone()[0]
# Update last seen toot id
new_last_seen_toot_id = -1
if all_pages:
new_last_seen_toot_id = all_pages[0]['id']
if new_last_seen_toot_id > last_seen_toot_id:
self.cursor.execute('insert into toot_cache values (?)', (new_last_seen_toot_id,))
self.conn.commit()
# Recurse in case the catch up took long enough for more toots to enter the public timeline
# Do this only once to be safe
if recurse:
self.replay_toots() # Recurse ONCE to catch up on any missing toots posted while doing initial catch up
def welcome_user(self, status):
''' Method that sets up toot and welcomes new users (method due to use in multiple places) '''
toot_id = status['id']
federated = '@' in status['account']['acct']
username = status['account']['acct']
timestamp = status['created_at']
visibility = status['visibility']
# Cache toot
self.cursor.execute('insert into toot_cache values (?)', (toot_id,))
self.conn.commit()
# Welcome any user who's posted publicly
if visibility == 'public' and not federated:
# Check if username has been seen for welcome
self.cursor.execute('select count(1) as found from welcome_cache where username = ?', (username,))
if self.cursor.fetchone()[0] > 0:
return
# Send welcome toot
toot(self.bot_config, username=username)
# Cache user to avoid duping welcome messages
self.cursor.execute('insert into welcome_cache values (?, ?)', (username, timestamp))
self.conn.commit()
def on_update(self, status):
'''A new status has appeared! 'status' is the parsed JSON dictionary
describing the status.'''
self.welcome_user(status)
def on_notification(self, notification):
'''A new notification. 'notification' is the parsed JSON dictionary
describing the notification.'''
# We don't care if notifications come through our bot / curation account
# Leave handling notifications/folow up to the admins and e-mail notifications
pass
def on_delete(self, status_id):
'''A status has been deleted. status_id is the status' integer ID.'''
# Remove the status from the toot_cache if we see a delete
self.cursor.execute('delete from toot_cache where toot_id = ?', (status_id,))
self.conn.commit()
def handle_heartbeat(self):
'''The server has sent us a keep-alive message. This callback may be
useful to carry out periodic housekeeping tasks, or just to confirm
that the connection is still open.'''
# Consistently/constantly trim the toot cache to the most recent seen toot
self.cursor.execute('delete from toot_cache where toot_id <= ((select max(toot_id) from toot_cache) - 1);')
self.conn.commit()
class CurationBot(StreamListener):
''' Implementation of the Mastodon.py StreamListener class for curation bot purposes '''
def __init__(self, bot_config):
StreamListener.__init__(self)
self.bot_config = bot_config
# Get access to cache
self.conn = sqlite3.connect(self.bot_config['curation']['cache_file'])
self.cursor = self.conn.cursor()
# Ensure cache table has been created
self.cursor.execute('''create table if not exists toot_cache (
toot_id integer primary key,
federated bool,
username varchar(2048),
is_reply bool,
is_boost bool,
toot_timestamp timestamp,
favorites integer,
boosts integer
);'''
)
self.conn.commit()
def __del__(self):
# Cleanup connection to sqlite database for rss cache
self.conn.close()
def on_update(self, status):
'''A new status has appeared! 'status' is the parsed JSON dictionary
describing the status.'''
if status['visibility'] == 'public':
toot_id = status['id']
federated = '@' in status['account']['acct']
username = status['account']['acct']
is_reply = status['in_reply_to_id'] is not None
is_boost = status['reblog'] is not None
timestamp = status['created_at']
favorites = status['favourites_count']
boosts = status['reblogs_count']
#tags = status['tags'] # TODO: Add support for tags? - Will need many to 1 relationship and a cross table
# Ensure a toot isn't cached twice for some odd reason
self.cursor.execute('select count(1) as found from toot_cache where toot_id = ?', (toot_id,))
if self.cursor.fetchone()[0] > 0:
return
# Cache toot
self.cursor.execute('insert into toot_cache values (?, ?, ?, ?, ?, ?, ?, ?)', (toot_id, federated, username, is_reply, is_boost, timestamp, favorites, boosts))
self.conn.commit()
def on_notification(self, notification):
'''A new notification. 'notification' is the parsed JSON dictionary
describing the notification.'''
# We don't care if notifications come through our bot / curation account
# Leave handling notifications/folow up to the admins and e-mail notifications
pass
def on_delete(self, status_id):
'''A status has been deleted. status_id is the status' integer ID.'''
# Remove the status from the toot_cache if we see a delete
self.cursor.execute('delete from toot_cache where toot_id = ?', (status_id,))
self.conn.commit()
def handle_heartbeat(self):
'''The server has sent us a keep-alive message. This callback may be
useful to carry out periodic housekeeping tasks, or just to confirm
that the connection is still open.'''
print('!!!!!!!!heartbeat!!!!!!!!')
print(' we should probably update statuses and whatnot here or at least do some housekeeping for old toots')

18
README.md

@ -17,7 +17,6 @@ This was designed with the admin in mind and functionality will grow over time.
- feedparser Python Module
- ruamel.yaml Python module
- argparse Python module (should be built in)
- configparser Python module (should be built in)
- sqlite3 Python module (should be built in)
# Installation
@ -36,7 +35,7 @@ You'll probably want to run the 2nd command on a schedule via cron for auto-toot
## RSS Toots
To cross post RSS articles take a look at 'configs/example_rss.yaml' and do the following
1. Run python ```.\toot_bod.py --config .\configs\example_rss.yaml init```
1. Run python ```.\toot_bot.py --config .\configs\example_rss.yaml init```
1. Run python ```.\toot_bot.py --config .\configs\example_rss.yaml rss```
The above will initialize the configuration (including login, you'll be prompted for user/password) then toot the various articles from the RSS feed. *BE CAREFUL*, the cache will be empty to start and you might spam your instance if there are a large number of articles in the feed.
@ -47,8 +46,18 @@ You'll probably want to run the 2nd command on a schedule via cron for cross pos
You can also implement custom logic for filtering RSS feeds, take a look at ```configs/example_rss_custom_logic.yaml``` and ```custom_logic/example_custom_rss_include.py``` for ideas.
## TOTP
If you're using a bot with an account that has 2FA / TOTP enabled you can pass the ```--totp``` option to ```init``` and ```login``` to make the init and login functions work correctly. There will be additional steps and you'll be prompted to complete the necessary steps.
## Welcome Bot
To welcome new users the first time they toot take a look at 'configs/example_welcome_config.yaml' and do the following
1. Run python ```.\toot_bot.py --config .\configs\example_welcome_config.yaml init```
1. Run python ```.\toot_bot.py --config .\configs\example_welcome_config.yaml welcome```
The above will initialize the configuration (including login, you'll be prompted for user/password) then toot a welcome message to any new users that toot and haven't been seen by the bot. It's a great way to say hello to new users of an instance. *BE CAREFUL*, the cache will be empty to start and you might spam your instance if there are a large number of existing toots.
Seriously, this is DANGEROUS if your instance has been around for a bit. You may want to try it in development first and/or create the sqlite database by hand and insert a row into toot_cache close to the current toot id of your instance.
Note: the welcome cache will be initialized the first time you run the welcome command
You'll probably want to run the 2nd command via the sample systemd unit file at ```configs/example_welcome_bot_service```.
## TOTP
If you're using a bot with an account that has 2FA / TOTP enabled you can pass the ```--totp``` option to ```init``` and ```login``` to make the init and login functions work correctly. There will be additional steps and you'll be prompted to complete the necessary steps.
@ -64,5 +73,4 @@ All code in this project is licensed under the GPLv3. The "GPLv3" file contains
## Documentation
All documentation is licensed under the Creative Commons Attribution, Non Commercial, Share Alike 4.0 International license (CC BY-NC-SA 4.0). The "CC_BY-NC-SA_4.0.html" file contains the full license.
![Creative Commons License](https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png)

58
Toot.py

@ -0,0 +1,58 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Adjust PyLint settings
# pylint: disable=C0301
''' Various Toot help method for bots '''
from mastodon import Mastodon
def toot(config, article=None, username=None):
''' Send a toot '''
# Setup Mastodon API
mastodon = Mastodon(
client_id=config['config']['client_cred_file'],
access_token=config['config']['user_cred_file'],
api_base_url=config['config']['api_base_url']
)
# Process main toot
cw_text = None
if 'cw_text' in config['toot']:
cw_text = config['toot']['cw_text']
# Replace RSS variables if they are present
if article is not None:
cw_text = cw_text.replace('[[ title ]]', article.title)
cw_text = cw_text.replace('[[ url ]]', article.link)
cw_text = cw_text.replace('[[ published ]]', article.published)
# Replace username variable if it's present
if username is not None:
cw_text = cw_text.replace('[[ username ]]', username)
toot_text = config['toot']['toot_text']
# Replace RSS varaibles if they are present
if article is not None:
toot_text = toot_text.replace('[[ title ]]', article.title)
toot_text = toot_text.replace('[[ url ]]', article.link)
toot_text = toot_text.replace('[[ published ]]', article.published)
# Replace username variable if it's present
if username is not None:
toot_text = toot_text.replace('[[ username ]]', username)
# Send main toot
parent_toot = mastodon.status_post(toot_text, visibility=config['toot']['visibility'], spoiler_text=cw_text)
parent_toot_id = parent_toot['id']
# Handle any sub-toots
if 'subtoots' in config['toot']:
for subtoot in config['toot']['subtoots']:
cw_text = None
if 'cw_text' in subtoot:
cw_text = subtoot['cw_text']
mastodon.status_post(subtoot['toot_text'], in_reply_to_id=parent_toot_id, visibility=subtoot['visibility'], spoiler_text=cw_text)
# Return the parent toot dict just in case it's needed elsewhere
return parent_toot

5
configs/example_generic_config.yaml

@ -21,7 +21,10 @@ toot:
cw_text: this is the CW text of the toot (optional)
toot_text: |
this is the body of the toot (required)
If welcome directive is specified the following special variables can be used as placeholders for information about toot that was seen
- [[ username ]]
If RSS directive is specified the following special variables can be used as placeholders for information about the RSS article
- [[ title ]]
- [[ url ]]

1
configs/example_rss.yaml

@ -7,7 +7,6 @@ config:
user_cred_file: /home/mastodon/bot/user_cred.secret
# Setup the RSS feed to watch
# This section is OPTIONAL
rss:
feed: https://instance.com/users/admin.atom
cache_file: /home/mastodon/bot/rss_cache.db # Path to sqlite database that caches seen articles

1
configs/example_rss_custom_logic.yaml

@ -7,7 +7,6 @@ config:
user_cred_file: /home/mastodon/bot/user_cred.secret
# Setup the RSS feed to watch
# This section is OPTIONAL
rss:
feed: https://instance.com/users/admin.atom
cache_file: /home/mastodon/bot/rss_cache.db # Path to sqlite database that caches seen articles

16
configs/example_welcome_bot.service

@ -0,0 +1,16 @@
# drop at /etc/systemd/system/mastodon_welcome_bot.service
[Unit]
Description=mastodon-welcome-bot
After=network.target
[Service]
Type=simple
User=mastodon
WorkingDirectory=/home/mastodon/bot
ExecStart=/usr/bin/python3 /home/mastodon/bot/toot_bot.py --config /home/mastodon/bot/welcome_bot.yaml welcome
TimeoutSec=15
Restart=always
[Install]
WantedBy=multi-user.target

26
configs/example_welcome_config.yaml

@ -0,0 +1,26 @@
# Setup general config for the bot (registered name, instance url, credential cache)
# This section is REQUIRED
config:
app_name: curation bot local and federated
api_base_url: https://instance.com
client_cred_file: /home/mastodon/bot/client_cred.secret
user_cred_file: /home/mastodon/bot/user_cred.secret
# Setup the cache file used for ensuring users aren't welcomed more than once
welcome:
cache_file: /home/mastodon/bot/welcome_cache.db
# Setup the toots that will be sent
# These static toots will ALWAYS be sent
# This section is REQUIRED
toot:
visibility: public # One of: public, unlisted, private, or direct
cw_text: Let's All Welcome @[[ username ]] to the community
toot_text: |
Welcome @[[ username ]]!
Welcome to instance.com, a growing community of people. If you're into ____ you've found the right instance. Our benevolent ______ is on a mission to create a great space for ______.
______ also use #______ and #______ heavily to surface content across the federverse.
There is also a great intro guide at http://ow.ly/X98j30cDwBw explaining Mastodon's quirks.

136
toot_bot.py

@ -4,7 +4,7 @@
# Adjust PyLint settings
# pylint: disable=C0301
''' Basic bot that toots '''
''' Various Mastodon bots and their implementations '''
import argparse
import sys
@ -12,78 +12,11 @@ import os
import getpass
import codecs
import sqlite3
from mastodon import Mastodon, StreamListener
from mastodon import Mastodon
import feedparser
from ruamel.yaml import YAML
class CurationBot(StreamListener):
''' Implementation of the Mastodon.py StreamListener class for curation bot purposes '''
def __init__(self, bot_config):
StreamListener.__init__(self)
self.bot_config = bot_config
# Get access to cache
self.conn = sqlite3.connect(self.bot_config['curation']['cache_file'])
self.cursor = self.conn.cursor()
# Ensure cache table has been created
self.cursor.execute('''create table if not exists toot_cache (
toot_id integer primary key,
federated bool,
username varchar(2048),
is_reply bool,
is_boost bool,
toot_timestamp timestamp,
favorites integer,
boosts integer
);'''
)
def __del__(self):
# Cleanup connection to sqlite database for rss cache
self.conn.close()
def on_update(self, status):
'''A new status has appeared! 'status' is the parsed JSON dictionary
describing the status.'''
if status['visibility'] == 'public':
toot_id = status['id']
federated = '@' in status['account']['acct']
username = status['account']['acct']
is_reply = status['in_reply_to_id'] is not None
is_boost = status['reblog'] is not None
timestamp = status['created_at']
favorites = status['favourites_count']
boosts = status['reblogs_count']
#tags = status['tags'] # TODO: Add support for tags? - Will need many to 1 relationship and a cross table
# Ensure a toot isn't cached twice for some odd reason
self.cursor.execute('select count(1) as found from toot_cache where toot_id = ?', (toot_id,))
if self.cursor.fetchone()[0] > 0:
pass
# Cache toot
self.cursor.execute('insert into toot_cache values (?, ?, ?, ?, ?, ?, ?, ?)', (toot_id, federated, username, is_reply, is_boost, timestamp, favorites, boosts))
self.conn.commit()
def on_notification(self, notification):
'''A new notification. 'notification' is the parsed JSON dictionary
describing the notification.'''
# We don't care if notifications come through our bot / curation account
# Leave handling notifications/folow up to the admins and e-mail notifications
pass
def on_delete(self, status_id):
'''A status has been deleted. status_id is the status' integer ID.'''
# Remove the status from the toot_cache if we see a delete
self.cursor.execute('delete from toot_cache where toot_id = ?', (status_id,))
self.conn.commit()
def handle_heartbeat(self):
'''The server has sent us a keep-alive message. This callback may be
useful to carry out periodic housekeeping tasks, or just to confirm
that the connection is still open.'''
print('!!!!!!!!heartbeat!!!!!!!!')
print(' we should probably update statuses and whatnot here or at least do some housekeeping for old toots')
from Toot import toot
from BotStreamListeners import CurationBot, WelcomeBot
def init(config):
''' Initialize the Mastdon API app and cache the API key
@ -138,45 +71,6 @@ def login(config):
to_file=config['config']['user_cred_file']
)
def toot(config, article=None):
''' Send a toot '''
# Setup Mastodon API
mastodon = Mastodon(
client_id=config['config']['client_cred_file'],
access_token=config['config']['user_cred_file'],
api_base_url=config['config']['api_base_url']
)
# Process main toot
cw_text = None
if 'cw_text' in config['toot']:
cw_text = config['toot']['cw_text']
if article is not None:
cw_text = cw_text.replace('[[ title ]]', article.title)
cw_text = cw_text.replace('[[ url ]]', article.link)
cw_text = cw_text.replace('[[ published ]]', article.published)
toot_text = config['toot']['toot_text']
if article is not None:
toot_text = toot_text.replace('[[ title ]]', article.title)
toot_text = toot_text.replace('[[ url ]]', article.link)
toot_text = toot_text.replace('[[ published ]]', article.published)
parent_toot = mastodon.status_post(toot_text, visibility=config['toot']['visibility'], spoiler_text=cw_text)
parent_toot_id = parent_toot['id']
# Handle any sub-toots
if 'subtoots' in config['toot']:
for subtoot in config['toot']['subtoots']:
cw_text = None
if 'cw_text' in subtoot:
cw_text = subtoot['cw_text']
mastodon.status_post(subtoot['toot_text'], in_reply_to_id=parent_toot_id, visibility=subtoot['visibility'], spoiler_text=cw_text)
# Return the parent toot dict just in case it's needed elsewhere
return parent_toot
# Default implementation on whether or not to include an RSS article
# Returns true and has the same method signature as what end users can setup for custom logic
def default_include_article(article):
@ -205,6 +99,7 @@ def rss(config):
# Ensure cache table has been created
cursor.execute('create table if not exists article_cache (id varchar(256) primary key);')
conn.commit()
# Run through all articles in feed
for entry in feed['entries']:
@ -237,6 +132,17 @@ def curate(config):
mastodon.public_stream(CurationBot(config))
def welcome(config):
''' Welcome new users to the instance '''
# Setup Mastodon API
mastodon = Mastodon(
client_id=config['config']['client_cred_file'],
access_token=config['config']['user_cred_file'],
api_base_url=config['config']['api_base_url']
)
mastodon.public_stream(WelcomeBot(config))
if __name__ == '__main__':
# Global CLI arguments/options
PARSER = argparse.ArgumentParser()
@ -255,6 +161,7 @@ if __name__ == '__main__':
TOOT_PARSER = SUBPARSERS.add_parser('toot', help='send configured toot')
RSS_PARSER = SUBPARSERS.add_parser('rss', help='cross post articles from an rss feed')
CURATE_PARSER = SUBPARSERS.add_parser('curation', help='Run the curation bot (service)')
WELCOME_PARSER = SUBPARSERS.add_parser('welcome', help='Run the welcome bot (service)')
# Parse CLI arguments
ARGS = PARSER.parse_args()
@ -306,6 +213,15 @@ if __name__ == '__main__':
if not os.path.exists(os.path.abspath(CONFIG['curation']['cache_file'])):
print('warning: curation cache_file file will be created')
curate(CONFIG)
# Deal with welcome command
if ARGS.command == 'welcome':
# Verify toot cache file folder exists
if not os.path.exists(os.path.abspath(os.path.split(CONFIG['welcome']['cache_file'])[0])):
sys.exit('welcome cache_file directory does not exist')
# Warn if welcome cache file doesn't exist
if not os.path.exists(os.path.abspath(CONFIG['welcome']['cache_file'])):
print('warning: welcome cache_file file will be created')
welcome(CONFIG)
# Deal with toot command
if ARGS.command == 'toot':
# Ensure main toot is <= 500 characters

Loading…
Cancel
Save