New Research: Supply Chain Attack on Axios Pulls Malicious Dependency from npm.Details →
Socket
Book a DemoSign in
Socket
Book a DemoSign in

rssbot

Package Overview
Dependencies
Maintainers
1
Versions
62
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

rssbot - pypi Package Compare versions

Comparing version
652
 to
653
+48
rssbot/brokers.py
# This file is placed in the Public Domain.
"client for a string"
class Fleet:
clients = {}
@staticmethod
def add(client):
Fleet.clients[repr(client)] = client
@staticmethod
def all():
return Fleet.clients.values()
@staticmethod
def announce(txt):
for client in Fleet.all():
client.announce(txt)
@staticmethod
def display(evt):
client = Fleet.get(evt.orig)
client.display(evt)
@staticmethod
def get(orig):
return Fleet.clients.get(orig, None)
@staticmethod
def say(orig, channel, txt):
client = Fleet.get(orig)
client.say(channel, txt)
@staticmethod
def shutdown():
for client in Fleet.all():
client.wait()
client.stop()
def __dir__():
return (
'Fleet',
)
+111
rssbot/caching.py
# This file is placed in the Public Domain.
"object for a string"
import json.decoder
import os
import threading
from .methods import deleted, search
from .objects import Object, update
from .serials import dump, load
from .workdir import cdir, fqn, getpath, j, long, store
from .utility import fntime
lock = threading.RLock()
class Cache:
objs = {}
@staticmethod
def add(path, obj):
Cache.objs[path] = obj
@staticmethod
def get(path):
return Cache.objs.get(path, None)
@staticmethod
def update(path, obj):
if path in Cache.objs:
update(Cache.objs[path], obj)
else:
Cache.add(path, obj)
def find(clz, selector=None, removed=False, matching=False):
clz = long(clz)
if selector is None:
selector = {}
for pth in fns(clz):
obj = Cache.get(pth)
if not obj:
obj = Object()
read(obj, pth)
Cache.add(pth, obj)
if not removed and deleted(obj):
continue
if selector and not search(obj, selector, matching):
continue
yield pth, obj
def fns(clz):
pth = store(clz)
for rootdir, dirs, _files in os.walk(pth, topdown=False):
for dname in dirs:
ddd = j(rootdir, dname)
for fll in os.listdir(ddd):
yield j(ddd, fll)
def last(obj, selector=None):
if selector is None:
selector = {}
result = sorted(
find(fqn(obj), selector),
key=lambda x: fntime(x[0])
)
res = ""
if result:
inp = result[-1]
update(obj, inp[-1])
res = inp[0]
return res
def read(obj, path):
with lock:
with open(path, "r", encoding="utf-8") as fpt:
try:
update(obj, load(fpt))
except json.decoder.JSONDecodeError as ex:
ex.add_note(path)
raise ex
def write(obj, path=None):
with lock:
if path is None:
path = getpath(obj)
cdir(path)
with open(path, "w", encoding="utf-8") as fpt:
dump(obj, fpt, indent=4)
Cache.update(path, obj)
return path
def __dir__():
return (
'Cache',
'find',
'last',
'read',
'write'
)
+78
rssbot/clients.py
# This file is placed in the Public Domain.
"client events"
import queue
import threading
import _thread
from .brokers import Fleet
from .handler import Handler
from .threads import launch
class Client(Handler):
def __init__(self):
Handler.__init__(self)
self.olock = threading.RLock()
self.oqueue = queue.Queue()
self.silent = True
Fleet.add(self)
def announce(self, txt):
if not self.silent:
self.raw(txt)
def display(self, event):
with self.olock:
for tme in sorted(event.result):
self.dosay(
event.channel,
event.result[tme]
)
def dosay(self, channel, txt):
self.say(channel, txt)
def raw(self, txt):
raise NotImplementedError("raw")
def say(self, channel, txt):
self.raw(txt)
class Output(Client):
def output(self):
while True:
event = self.oqueue.get()
if event is None:
self.oqueue.task_done()
break
self.display(event)
self.oqueue.task_done()
def start(self):
launch(self.output)
super().start()
def stop(self):
self.oqueue.put(None)
super().stop()
def wait(self):
try:
self.oqueue.join()
except Exception:
_thread.interrupt_main()
def __dir__():
return (
'Client',
'Output'
)
+18
rssbot/modules/flt.py
# This file is placed in the Public Domain.
from ..clients import Fleet
from ..methods import fmt
from ..threads import name
def flt(event):
if event.args:
clts = Fleet.all()
index = int(event.args[0])
if index < len(clts):
event.reply(fmt(list(Fleet.all())[index], empty=True))
else:
event.reply(f"only {len(clts)} clients in fleet.")
return
event.reply(' | '.join([name(o) for o in Fleet.all()]))
+32
rssbot/modules/fnd.py
# This file is placed in the Public Domain.
"find"
import time
from ..caching import find
from ..methods import fmt
from ..utility import elapsed, fntime
from ..workdir import long, skel, types
def fnd(event):
skel()
if not event.rest:
res = sorted([x.split('.')[-1].lower() for x in types()])
if res:
event.reply(",".join(res))
else:
event.reply("no data yet.")
return
otype = event.args[0]
clz = long(otype)
nmr = 0
for fnm, obj in list(find(clz, event.gets)):
event.reply(f"{nmr} {fmt(obj)} {elapsed(time.time()-fntime(fnm))}")
nmr += 1
if not nmr:
event.reply("no result")
+15
rssbot/modules/lst.py
# This file is been placed in the Public Domain.
"available types"
from ..workdir import types
def lst(event):
tps = types()
if tps:
event.reply(",".join([x.split(".")[-1].lower() for x in tps]))
else:
event.reply("no data yet.")
+11
rssbot/modules/mod.py
# This file is placed in the Public Domain.
"show modules"
from ..package import modules
def mod(event):
event.reply(",".join(modules()))
+19
rssbot/modules/sil.py
# This file is placed in the Public Domain.
"enable silence mode"
from ..brokers import Fleet
def sil(event):
bot = Fleet.get(event.orig)
bot.silent = True
event.reply("ok")
def lou(event):
bot = Fleet.get(event.orig)
bot.silent = False
event.reply("ok")
+36
rssbot/modules/thr.py
# This file is placed in the Public Domain.
"show running threads"
import threading
import time
from ..utility import elapsed
STARTTIME = time.time()
def thr(event):
result = []
for thread in sorted(threading.enumerate(), key=lambda x: x.name):
if str(thread).startswith("<_"):
continue
if getattr(thread, "state", None) and getattr(thread, "sleep", None):
uptime = thread.sleep - int(time.time() - thread.state["latest"])
elif getattr(thread, "starttime", None):
uptime = int(time.time() - thread.starttime)
else:
uptime = int(time.time() - STARTTIME)
result.append((uptime, thread.name))
res = []
for uptime, txt in sorted(result, key=lambda x: x[0]):
lap = elapsed(uptime)
res.append(f"{txt}/{lap}")
if res:
event.reply(" ".join(res))
else:
event.reply("no threads")
+17
rssbot/modules/upt.py
# This file is placed in the Public Domain.
"uptime"
import time
from ..utility import elapsed
STARTTIME = time.time()
def upt(event):
event.reply(elapsed(time.time()-STARTTIME))
+4
rssbot/network/__init__.py
# This file is placed in the Public Domain.
"network"
+115
rssbot/package.py
# This file is placed in the Public Domain.
"module management"
import inspect
import logging
import os
import sys
import threading
import _thread
from .threads import launch
from .utility import importer, md5sum
from .workdir import Workdir, j, moddir
NAME = Workdir.name
PATH = os.path.dirname(inspect.getfile(Workdir))
lock = threading.RLock()
class Mods:
debug = False
dirs = {}
md5s = {}
@staticmethod
def dir(name, path=None):
if path is not None:
Mods.dirs[name] = path
else:
Mods.dirs[NAME + "." + name] = j(PATH, name)
def getmod(name):
for nme, path in Mods.dirs.items():
mname = nme + "." + name
module = sys.modules.get(mname, None)
if module:
return module
pth = j(path, f"{name}.py")
if Mods.md5s:
if os.path.exists(pth) and name != "tbl":
md5 = Mods.md5s.get(name, None)
if md5sum(pth) != md5:
file = pth.split(os.sep)[-1]
logging.info("md5 error %s", file)
mod = importer(mname, pth)
if mod:
return mod
def inits(names):
modz = []
for name in modules():
if name not in names:
continue
try:
module = getmod(name)
if module and "init" in dir(module):
thr = launch(module.init)
modz.append((module, thr))
except Exception as ex:
logging.exception(ex)
_thread.interrupt_main()
return modz
def modules():
mods = []
for name, path in Mods.dirs.items():
if not os.path.exists(path):
continue
mods.extend([
x[:-3] for x in os.listdir(path)
if x.endswith(".py") and not x.startswith("__")
])
return sorted(mods)
def setdirs(network=False, mods=False):
Mods.dir("modules")
Mods.dir("local", moddir())
if network:
Mods.dir("network")
if mods:
Mods.dir("mods", "mods")
def sums(checksum):
tbl = getmod("tbl")
if not tbl:
logging.info("no table")
return
if "MD5" in dir(tbl):
Mods.md5s.update(tbl.MD5)
def __dir__():
return (
'Mods',
'getmod',
'importer',
'inits',
'md5sum',
'modules',
'setdirs',
'sums'
)
+46
rssbot/serials.py
# This file is placed in the Public Domain.
"json serializer"
from json import JSONEncoder
from json import dump as jdump
from json import dumps as jdumps
from json import load as load
from json import loads as loads
class Encoder(JSONEncoder):
def default(self, o):
if isinstance(o, dict):
return o.items()
if isinstance(o, list):
return iter(o)
try:
return JSONEncoder.default(self, o)
except TypeError:
try:
return vars(o)
except TypeError:
return repr(o)
def dump(obj, fp, *args, **kw):
kw["cls"] = Encoder
jdump(obj, fp, *args, **kw)
def dumps(obj, *args, **kw):
kw["cls"] = Encoder
return jdumps(obj, *args, **kw)
def __dir__():
return (
'dump',
'dumps',
'load',
'loads'
)
+111
rssbot/threads.py
# This file is placed in the Public Domain.
"non-blocking"
import logging
import queue
import threading
import time
import _thread
from .methods import name
class Thread(threading.Thread):
def __init__(self, func, *args, daemon=True, **kwargs):
super().__init__(None, self.run, None, (), daemon=daemon)
self.name = kwargs.get("name", name(func))
self.queue = queue.Queue()
self.result = None
self.starttime = time.time()
self.stopped = threading.Event()
self.queue.put((func, args))
def __iter__(self):
return self
def __next__(self):
yield from dir(self)
def join(self, timeout=None):
result = None
try:
super().join(timeout)
result = self.result
except (KeyboardInterrupt, EOFError):
_thread.interrupt_main()
return result
def run(self):
func, args = self.queue.get()
try:
self.result = func(*args)
except (KeyboardInterrupt, EOFError):
_thread.interrupt_main()
except Exception as ex:
logging.exception(ex)
_thread.interrupt_main()
class Timy(threading.Timer):
def __init__(self, sleep, func, *args, **kwargs):
super().__init__(sleep, func)
self.name = kwargs.get("name", name(func))
self.sleep = sleep
self.state = {}
self.state["latest"] = time.time()
self.state["starttime"] = time.time()
self.starttime = time.time()
class Timed:
def __init__(self, sleep, func, *args, thrname="", **kwargs):
self.args = args
self.func = func
self.kwargs = kwargs
self.sleep = sleep
self.name = thrname or kwargs.get("name", name(func))
self.target = time.time() + self.sleep
self.timer = None
def run(self):
self.timer.latest = time.time()
self.func(*self.args)
def start(self):
self.kwargs["name"] = self.name
timer = Timy(self.sleep, self.run, *self.args, **self.kwargs)
timer.start()
self.timer = timer
def stop(self):
if self.timer:
self.timer.cancel()
class Repeater(Timed):
def run(self):
launch(self.start)
super().run()
def launch(func, *args, **kwargs):
thread = Thread(func, *args, **kwargs)
thread.start()
return thread
def __dir__():
return (
'Repeater',
'Thread',
'launch',
'name'
)
+164
rssbot/utility.py
# This file is placed in the Public Domain.
"utilities"
import hashlib
import importlib.util
import logging
import os
import sys
import time
import _thread
FORMATS = [
"%Y-%M-%D %H:%M:%S",
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%d",
"%d-%m-%Y",
"%d-%m",
"%m-%d",
]
LEVELS = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'warn': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL,
}
class Formatter(logging.Formatter):
def format(self, record):
record.module = record.module.upper()
return logging.Formatter.format(self, record)
def elapsed(seconds, short=True):
txt = ""
nsec = float(seconds)
if nsec < 1:
return f"{nsec:.2f}s"
yea = 365 * 24 * 60 * 60
week = 7 * 24 * 60 * 60
nday = 24 * 60 * 60
hour = 60 * 60
minute = 60
yeas = int(nsec / yea)
nsec -= yeas * yea
weeks = int(nsec / week)
nsec -= weeks * week
nrdays = int(nsec / nday)
nsec -= nrdays * nday
hours = int(nsec / hour)
nsec -= hours * hour
minutes = int(nsec / minute)
nsec -= int(minute * minutes)
sec = int(nsec)
if yeas:
txt += f"{yeas}y"
if weeks:
nrdays += weeks * 7
if nrdays:
txt += f"{nrdays}d"
if short and txt:
return txt.strip()
if hours:
txt += f"{hours}h"
if minutes:
txt += f"{minutes}m"
if sec:
txt += f"{sec}s"
txt = txt.strip()
return txt
def extract_date(daystr):
daystr = daystr.encode('utf-8', 'replace').decode("utf-8")
res = time.time()
for fmat in FORMATS:
try:
res = time.mktime(time.strptime(daystr, fmat))
break
except ValueError:
pass
return res
def fntime(daystr):
datestr = " ".join(daystr.split(os.sep)[-2:])
datestr = datestr.replace("_", " ")
if "." in datestr:
datestr, rest = datestr.rsplit(".", 1)
else:
rest = ""
timed = time.mktime(time.strptime(datestr, "%Y-%m-%d %H:%M:%S"))
if rest:
timed += float("." + rest)
return float(timed)
def importer(name, pth):
if not os.path.exists(pth):
return
try:
spec = importlib.util.spec_from_file_location(name, pth)
if not spec or not spec.loader:
return
mod = importlib.util.module_from_spec(spec)
if not mod:
return
sys.modules[name] = mod
spec.loader.exec_module(mod)
logging.info("load %s", pth)
return mod
except Exception as ex:
logging.exception(ex)
_thread.interrupt_main()
def level(loglevel="debug"):
if loglevel != "none":
datefmt = "%H:%M:%S"
format_short = "%(module).3s %(message)-76s"
ch = logging.StreamHandler()
ch.setLevel(LEVELS.get(loglevel))
formatter = Formatter(fmt=format_short, datefmt=datefmt)
ch.setFormatter(formatter)
logger = logging.getLogger()
logger.addHandler(ch)
def md5sum(path):
with open(path, "r", encoding="utf-8") as file:
txt = file.read().encode("utf-8")
return hashlib.md5(txt).hexdigest()
def spl(txt):
try:
result = txt.split(",")
except (TypeError, ValueError):
result = [
txt,
]
return [x for x in result if x]
def __dir__():
return (
'cdir',
'elapsed',
'extract_date',
'fntime',
'importer',
'level',
'md5sum',
'spl'
)
+118
rssbot/workdir.py
# This file is placed in the Public Domain.
"working directory"
import datetime
import os
import pathlib
j = os.path.join
class Workdir:
name = __file__.rsplit(os.sep, maxsplit=2)[-2]
wdr = ""
def cdir(path):
pth = pathlib.Path(path)
pth.parent.mkdir(parents=True, exist_ok=True)
def fqn(obj):
kin = str(type(obj)).split()[-1][1:-2]
if kin == "type":
kin = f"{obj.__module__}.{obj.__name__}"
return kin
def getpath(obj):
return store(ident(obj))
def ident(obj):
return j(fqn(obj), *str(datetime.datetime.now()).split())
def long(name):
split = name.split(".")[-1].lower()
res = name
for names in types():
if split == names.split(".")[-1].lower():
res = names
break
return res
def moddir():
assert Workdir.wdr
return j(Workdir.wdr, "mods")
def pidfile(filename):
if os.path.exists(filename):
os.unlink(filename)
path2 = pathlib.Path(filename)
path2.parent.mkdir(parents=True, exist_ok=True)
with open(filename, "w", encoding="utf-8") as fds:
fds.write(str(os.getpid()))
def pidname(name):
assert Workdir.wdr
return j(Workdir.wdr, f"{name}.pid")
def setwd(name, path=""):
path = path or os.path.expanduser(f"~/.{name}")
Workdir.wdr = Workdir.wdr or path
skel()
def skel():
result = ""
if not os.path.exists(store()):
pth = pathlib.Path(store())
pth.mkdir(parents=True, exist_ok=True)
pth = pathlib.Path(moddir())
pth.mkdir(parents=True, exist_ok=True)
result = str(pth)
return result
def store(pth=""):
assert Workdir.wdr
return j(Workdir.wdr, "store", pth)
def strip(pth, nmr=2):
return j(pth.split(os.sep)[-nmr:])
def types():
skel()
return os.listdir(store())
def __dir__():
return (
'Workdir',
'cdir',
'fqn',
'getpath',
'ident',
'j',
'long',
'moddir',
'pidfile',
'pidname',
'setwd',
'skel',
'store',
'strip',
'types'
)
+2
-2
PKG-INFO
Metadata-Version: 2.4
Name: rssbot
Version: 652
Summary: 24/7 Feed Fetcher.
Version: 653
Summary: 24/7 Feed Fetcher
Author-email: Bart Thate <rssbotd@gmail.com>

@@ -6,0 +6,0 @@ License-Expression: Unlicense

+4
-3
pyproject.toml

@@ -11,4 +11,4 @@ [build-system]

name = "rssbot"
description = "24/7 Feed Fetcher."
version = "652"
description = "24/7 Feed Fetcher"
version = "653"
authors = [

@@ -36,3 +36,4 @@ {name = "Bart Thate",email = "rssbotd@gmail.com"},

"rssbot",
"rssbot.modules"
"rssbot.modules",
"rssbot.network"
]
+2
-2
rssbot.egg-info/PKG-INFO
Metadata-Version: 2.4
Name: rssbot
Version: 652
Summary: 24/7 Feed Fetcher.
Version: 653
Summary: 24/7 Feed Fetcher
Author-email: Bart Thate <rssbotd@gmail.com>

@@ -6,0 +6,0 @@ License-Expression: Unlicense

+16
-3
rssbot.egg-info/SOURCES.txt

@@ -5,2 +5,5 @@ README.rst

bin/rssbot
rssbot/brokers.py
rssbot/caching.py
rssbot/clients.py
rssbot/command.py

@@ -10,4 +13,7 @@ rssbot/handler.py

rssbot/objects.py
rssbot/persist.py
rssbot/runtime.py
rssbot/package.py
rssbot/serials.py
rssbot/threads.py
rssbot/utility.py
rssbot/workdir.py
rssbot.egg-info/PKG-INFO

@@ -17,5 +23,12 @@ rssbot.egg-info/SOURCES.txt

rssbot.egg-info/top_level.txt
rssbot/modules/flt.py
rssbot/modules/fnd.py
rssbot/modules/irc.py
rssbot/modules/lst.py
rssbot/modules/mod.py
rssbot/modules/rss.py
rssbot/modules/tbl.py
rssbot/modules/sil.py
rssbot/modules/thr.py
rssbot/modules/upt.py
rssbot/network/__init__.py
tests/test_none.py
+12
-161
rssbot/command.py
# This file is placed in the Public Domain.
"commands"
"write your own commands"
import hashlib
import importlib
import importlib.util
import inspect
import logging
import os
import sys
import threading
import _thread
from .methods import j, rlog, spl
from .handler import Fleet
from .brokers import Fleet
from .methods import parse
from .package import getmod, modules
lock = threading.RLock()
class Commands:
cmds = {}
debug = False
md5s = {}
mod = j(os.path.dirname(__file__), "modules")
package = __name__.split(".")[0] + "." + "modules"
names = {}

@@ -47,10 +33,6 @@

name = Commands.names.get(cmd, None)
if not name:
return
module = getmod(name)
if not module:
return
scan(module)
if Commands.debug:
module.DEBUG = True
if name:
module = getmod(name)
if module:
scan(module)
return Commands.cmds.get(cmd, None)

@@ -68,30 +50,2 @@

"modules"
def getmod(name, path=None):
with lock:
mname = Commands.package + "." + name
module = sys.modules.get(mname, None)
if module:
return module
if not path:
path = Commands.mod
pth = j(path, f"{name}.py")
if not os.path.exists(pth):
return
if name != "tbl" and md5sum(pth) != Commands.md5s.get(name, None):
rlog("warn", f"md5 error on {pth.split(os.sep)[-1]}")
return importer(mname, pth)
def modules():
if not os.path.exists(Commands.mod):
return {}
return {
x[:-3] for x in os.listdir(Commands.mod)
if x.endswith(".py") and not x.startswith("__")
}
def scan(module):

@@ -105,6 +59,6 @@ for key, cmdz in inspect.getmembers(module, inspect.isfunction):

def scanner(names=None):
def scanner(names=[]):
res = []
for nme in sorted(modules()):
if names and nme not in spl(names):
if names and nme not in names:
continue

@@ -120,12 +74,5 @@ module = getmod(nme)

def table(checksum=""):
pth = j(Commands.mod, "tbl.py")
if os.path.exists(pth):
if checksum and md5sum(pth) != checksum:
rlog("warn", "table checksum error.")
tbl = getmod("tbl")
if tbl:
if "NAMES" in dir(tbl):
Commands.names.update(tbl.NAMES)
if "MD5" in dir(tbl):
Commands.md5s.update(tbl.MD5)
if tbl and "NAMES" in dir(tbl):
Commands.names.update(tbl.NAMES)
else:

@@ -135,93 +82,2 @@ scanner()

"utilities"
def importer(name, pth):
try:
spec = importlib.util.spec_from_file_location(name, pth)
if not spec:
rlog("info", f"misiing {pth}")
return
module = importlib.util.module_from_spec(spec)
if not module:
rlog("info", f"{pth} not importable")
return
sys.modules[name] = module
spec.loader.exec_module(module)
rlog("info", f"load {pth}")
return module
except Exception as ex:
logging.exception(ex)
_thread.interrupt_main()
def md5sum(path):
with open(path, "r", encoding="utf-8") as file:
txt = file.read().encode("utf-8")
return hashlib.md5(txt).hexdigest()
def parse(obj, txt=None):
if txt is None:
if "txt" in dir(obj):
txt = obj.txt
else:
txt = ""
args = []
obj.args = getattr(obj, "args", [])
obj.cmd = getattr(obj, "cmd", "")
obj.gets = getattr(obj, "gets", "")
obj.index = getattr(obj, "index", None)
obj.inits = getattr(obj, "inits", "")
obj.mod = getattr(obj, "mod", "")
obj.opts = getattr(obj, "opts", "")
obj.result = getattr(obj, "result", "")
obj.sets = getattr(obj, "sets", {})
obj.silent = getattr(obj, "silent", "")
obj.txt = txt or getattr(obj, "txt", "")
obj.otxt = obj.txt or getattr(obj, "otxt", "")
_nr = -1
for spli in obj.otxt.split():
if spli.startswith("-"):
try:
obj.index = int(spli[1:])
except ValueError:
obj.opts += spli[1:]
continue
if "-=" in spli:
key, value = spli.split("-=", maxsplit=1)
obj.silent[key] = value
obj.gets[key] = value
continue
if "==" in spli:
key, value = spli.split("==", maxsplit=1)
obj.gets[key] = value
continue
if "=" in spli:
key, value = spli.split("=", maxsplit=1)
if key == "mod":
if obj.mod:
obj.mod += f",{value}"
else:
obj.mod = value
continue
obj.sets[key] = value
continue
_nr += 1
if _nr == 0:
obj.cmd = spli
continue
args.append(spli)
if args:
obj.args = args
obj.txt = obj.cmd or ""
obj.rest = " ".join(obj.args)
obj.txt = obj.cmd + " " + obj.rest
else:
obj.txt = obj.cmd or ""
"interface"
def __dir__():

@@ -231,7 +87,2 @@ return (

'command',
'getmod',
'importer',
'md5sum',
'modules',
'parse',
'scan',

@@ -238,0 +89,0 @@ 'scanner',

+11
-159
rssbot/handler.py
# This file is placed in the Public Domain.
"event handler"
"handle events"

@@ -13,4 +13,3 @@

from .methods import fqn
from .runtime import launch
from .threads import launch

@@ -22,14 +21,9 @@

self.cbs = {}
self.type = fqn(self)
self.queue = queue.Queue()
self.ready = threading.Event()
self.stopped = threading.Event()
def available(self, event):
return event.type in self.cbs
def callback(self, event):
func = self.cbs.get(event.type, None)
if func:
event._thr = launch(func, event, name=event.txt and event.txt.split()[0])
name = event.txt and event.txt.split()[0]
event._thr = launch(func, event, name=name)
else:

@@ -39,6 +33,6 @@ event.ready()

def loop(self):
while not self.stopped.is_set():
while True:
try:
event = self.poll()
if event is None or self.stopped.is_set():
if event is None:
break

@@ -56,145 +50,12 @@ event.orig = repr(self)

def register(self, typ, cbs):
self.cbs[typ] = cbs
def register(self, type, callback):
self.cbs[type] = callback
def start(self, daemon=True):
self.stopped.clear()
launch(self.loop, daemon=daemon)
def start(self):
launch(self.loop)
def stop(self):
self.stopped.set()
self.queue.put(None)
def wait(self):
pass
"clients"
class Client(Handler):
def __init__(self):
Handler.__init__(self)
self.olock = threading.RLock()
Fleet.add(self)
def announce(self, txt):
pass
def display(self, event):
with self.olock:
for tme in sorted(event.result):
self.dosay(event.channel, event.result[tme])
def dosay(self, channel, txt):
self.say(channel, txt)
def raw(self, txt):
raise NotImplementedError("raw")
def say(self, channel, txt):
self.raw(txt)
class Output(Client):
def __init__(self):
Client.__init__(self)
self.olock = threading.RLock()
self.oqueue = queue.Queue()
self.ostop = threading.Event()
def oput(self, event):
self.oqueue.put(event)
def output(self):
while not self.ostop.is_set():
event = self.oqueue.get()
if event is None:
self.oqueue.task_done()
break
self.display(event)
self.oqueue.task_done()
def start(self, daemon=True):
self.ostop.clear()
launch(self.output, daemon=daemon)
super().start()
def stop(self):
self.ostop.set()
self.oqueue.put(None)
super().stop()
def wait(self):
try:
self.oqueue.join()
except Exception:
_thread.interrupt_main()
"list of clients"
class Fleet:
clients = {}
@staticmethod
def add(client):
Fleet.clients[repr(client)] = client
@staticmethod
def all():
return list(Fleet.clients.values())
@staticmethod
def announce(txt):
for client in Fleet.all():
client.announce(txt)
@staticmethod
def dispatch(evt):
client = Fleet.get(evt.orig)
client.put(evt)
@staticmethod
def display(evt):
client = Fleet.get(evt.orig)
client.display(evt)
@staticmethod
def first():
clt = list(Fleet.all())
res = None
if clt:
res = clt[0]
return res
@staticmethod
def get(orig):
return Fleet.clients.get(orig, None)
@staticmethod
def say(orig, channel, txt):
client = Fleet.get(orig)
if client:
client.say(channel, txt)
@staticmethod
def shutdown():
for client in Fleet.all():
client.stop()
@staticmethod
def wait():
time.sleep(0.1)
for client in Fleet.all():
client.wait()
"event"
class Event:

@@ -214,5 +75,2 @@

def done(self):
self.reply("ok")
def ready(self):

@@ -228,3 +86,3 @@ self._ready.set()

if self._thr:
self._thr.join()
self._thr.join(timeout)
except (KeyboardInterrupt, EOFError):

@@ -234,12 +92,6 @@ _thread.interrupt_main()

"interface"
def __dir__():
return (
'Client',
'Event',
'Fleet',
'Handler'
'Output'
)
+86
-142
rssbot/methods.py
# This file is placed in the Public Domain.
"methods"
"object as the first argument"
import hashlib
import importlib
import importlib.util
import logging
import os
import sys
import time
import _thread
from .objects import items, keys
j = os.path.join
def deleted(obj):
return "__deleted__" in dir(obj) and obj.__deleted__

@@ -45,7 +36,5 @@

def fmt(obj, args=None, skip=None, plain=False, empty=False, newline=False):
if args is None:
def fmt(obj, args=[], skip=[], plain=False, empty=False):
if not args:
args = keys(obj)
if skip is None:
skip = []
txt = ""

@@ -66,20 +55,87 @@ for key in args:

txt += f'{key}="{value}" '
elif isinstance(value, (int, float, dict, bool, list)):
txt += f"{key}={value} "
else:
txt += f"{key}={value} "
if newline:
txt += "\n"
txt += f"{key}={name(value, True)} "
return txt.strip()
def fqn(obj):
kin = str(type(obj)).split()[-1][1:-2]
if kin == "type":
kin = f"{obj.__module__}.{obj.__name__}"
return kin
def name(obj, short=False):
typ = type(obj)
res = ""
if "__builtins__" in dir(typ):
res = obj.__name__
elif "__self__" in dir(obj):
res = f"{obj.__self__.__class__.__name__}.{obj.__name__}"
elif "__class__" in dir(obj) and "__name__" in dir(obj):
res = f"{obj.__class__.__name__}.{obj.__name__}"
elif "__class__" in dir(obj):
res = f"{obj.__class__.__module__}.{obj.__class__.__name__}"
elif "__name__" in dir(obj):
res = f"{obj.__class__.__name__}.{obj.__name__}"
if short:
res = res.split(".")[-1]
return res
def parse(obj, txt=""):
if not txt:
if "txt" in dir(obj):
txt = obj.txt
args = []
obj.args = getattr(obj, "args", [])
obj.cmd = getattr(obj, "cmd", "")
obj.gets = getattr(obj, "gets", "")
obj.index = getattr(obj, "index", None)
obj.inits = getattr(obj, "inits", "")
obj.mod = getattr(obj, "mod", "")
obj.opts = getattr(obj, "opts", "")
obj.result = getattr(obj, "result", "")
obj.sets = getattr(obj, "sets", {})
obj.silent = getattr(obj, "silent", "")
obj.txt = txt or getattr(obj, "txt", "")
obj.otxt = obj.txt or getattr(obj, "otxt", "")
_nr = -1
for spli in obj.otxt.split():
if spli.startswith("-"):
try:
obj.index = int(spli[1:])
except ValueError:
obj.opts += spli[1:]
continue
if "-=" in spli:
key, value = spli.split("-=", maxsplit=1)
obj.silent[key] = value
obj.gets[key] = value
continue
if "==" in spli:
key, value = spli.split("==", maxsplit=1)
obj.gets[key] = value
continue
if "=" in spli:
key, value = spli.split("=", maxsplit=1)
if key == "mod":
if obj.mod:
obj.mod += f",{value}"
else:
obj.mod = value
continue
obj.sets[key] = value
continue
_nr += 1
if _nr == 0:
obj.cmd = spli
continue
args.append(spli)
if args:
obj.args = args
obj.txt = obj.cmd or ""
obj.rest = " ".join(obj.args)
obj.txt = obj.cmd + " " + obj.rest
else:
obj.txt = obj.cmd or ""
def search(obj, selector, matching=False):
res = False
if not selector:
return res
for key, value in items(selector):

@@ -91,3 +147,3 @@ val = getattr(obj, key, None)

res = True
elif str(value).lower() in str(val).lower() or value == "match":
elif str(value).lower() in str(val).lower():
res = True

@@ -100,121 +156,9 @@ else:

"utilities"
def elapsed(seconds, short=True):
txt = ""
nsec = float(seconds)
if nsec < 1:
return f"{nsec:.2f}s"
yea = 365 * 24 * 60 * 60
week = 7 * 24 * 60 * 60
nday = 24 * 60 * 60
hour = 60 * 60
minute = 60
yeas = int(nsec / yea)
nsec -= yeas * yea
weeks = int(nsec / week)
nsec -= weeks * week
nrdays = int(nsec / nday)
nsec -= nrdays * nday
hours = int(nsec / hour)
nsec -= hours * hour
minutes = int(nsec / minute)
nsec -= int(minute * minutes)
sec = int(nsec)
if yeas:
txt += f"{yeas}y"
if weeks:
nrdays += weeks * 7
if nrdays:
txt += f"{nrdays}d"
if short and txt:
return txt.strip()
if hours:
txt += f"{hours}h"
if minutes:
txt += f"{minutes}m"
if sec:
txt += f"{sec}s"
txt = txt.strip()
return txt
def extract_date(daystr):
daystr = daystr.encode('utf-8', 'replace').decode("utf-8")
res = time.time()
for format in FORMATS:
try:
res = time.mktime(time.strptime(daystr, format))
break
except ValueError:
pass
return res
def level(loglevel="debug"):
if loglevel != "none":
format_short = "%(asctime)-8s %(message)-80s"
datefmt = "%H:%M:%S"
logging.basicConfig(datefmt=datefmt, format=format_short, force=True)
logging.getLogger().setLevel(LEVELS.get(loglevel))
def rlog(loglevel, txt, ignore=None):
if ignore is None:
ignore = []
for ign in ignore:
if ign in str(txt):
return
logging.log(LEVELS.get(loglevel), txt)
def spl(txt):
try:
result = txt.split(",")
except (TypeError, ValueError):
result = [
txt,
]
return [x for x in result if x]
"data"
FORMATS = [
"%Y-%M-%D %H:%M:%S",
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%d",
"%d-%m-%Y",
"%d-%m",
"%m-%d",
]
LEVELS = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'warn': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL,
}
"interface"
def __dir__():
return (
'deleted',
'edit',
'elapsed',
'extract_date',
'fmt',
'fqn',
'j',
'level',
'rlog',
'search',
'spl'
'parse',
'search'
)
+33
-22
rssbot/modules/irc.py

@@ -8,2 +8,3 @@ # This file is placed in the Public Domain.

import base64
import logging
import os

@@ -17,9 +18,12 @@ import socket

from rssbot.command import command
from rssbot.handler import Event as IEvent
from rssbot.handler import Fleet, Output
from rssbot.methods import edit, fmt, rlog
from rssbot.objects import Object, keys
from rssbot.persist import Workdir, getpath, last, write
from rssbot.runtime import launch
from ..brokers import Fleet
from ..caching import last, write
from ..clients import Output
from ..command import command
from ..handler import Event as IEvent
from ..methods import edit, fmt
from ..objects import Object, keys
from ..threads import launch
from ..utility import LEVELS
from ..workdir import Workdir, getpath

@@ -41,3 +45,3 @@

if irc.events.joined.is_set():
rlog("warn", f"irc {fmt(irc.cfg, skip=["password", "realname", "username"])} channels {",".join(irc.channels)}")
logging.warning(fmt(irc.cfg, skip=["password", "realname", "username"]))
else:

@@ -48,2 +52,11 @@ irc.stop()

def rlog(loglevel, txt, ignore=None):
if ignore is None:
ignore = []
for ign in ignore:
if ign in str(txt):
return
logging.log(LEVELS.get(loglevel), txt)
class Config:

@@ -119,2 +132,3 @@

self.events.ready = threading.Event()
self.silent = False
self.sock = None

@@ -153,3 +167,3 @@ self.state = Object()

if self.cfg.password:
rlog("debug", "using SASL")
logging.debug("using SASL")
self.cfg.sasl = True

@@ -174,6 +188,3 @@ self.cfg.port = "6697"

self.events.connected.set()
rlog(
"debug",
f"connected {self.cfg.server}:{self.cfg.port} {self.cfg.channel}",
)
logging.debug("connected %s:%s channel %s", self.cfg.server, self.cfg.port, self.cfg.channel)
return True

@@ -243,3 +254,3 @@ return False

self.state.error = str(ex)
rlog("debug", str(type(ex)) + " " + str(ex))
logging.debug("%s", str(type(ex)) + " " + str(ex))
time.sleep(self.cfg.sleep)

@@ -300,3 +311,3 @@

def keep(self):
while not self.stopped.is_set():
while True:
if self.state.stopkeep:

@@ -329,3 +340,3 @@ self.state.stopkeep = False

rawstr = rawstr.replace("\001", "")
rlog("debug", txt, IGNORE)
logging.debug(txt)
obj = Event()

@@ -405,3 +416,3 @@ obj.args = []

self.state.error = str(type(ex)) + " " + str(ex)
rlog("debug", self.state.error)
logging.debug(self.state.error)
self.state.pongcheck = True

@@ -418,3 +429,3 @@ self.stop()

txt = txt.rstrip()
rlog("debug", txt, IGNORE)
rlog("info", txt, IGNORE)
txt = txt[:500]

@@ -434,3 +445,3 @@ txt += "\r\n"

) as ex:
rlog("debug", str(type(ex)) + " " + str(ex))
logging.debug("%s", str(type(ex)) + " " + str(ex))
self.events.joined.set()

@@ -446,3 +457,3 @@ self.state.nrerror += 1

def reconnect(self):
rlog("debug", f"reconnecting {self.cfg.server:self.cfg.port}")
logging.debug("reconnecting %s:%s", self.cfg.server, self.cfg.port)
self.disconnect()

@@ -531,3 +542,3 @@ self.events.connected.clear()

bot.state.error = evt.txt
rlog("debug", fmt(evt))
logging.debug(fmt(evt))

@@ -593,3 +604,3 @@

bot = Fleet.get(evt.orig)
rlog("debug", f"quit from {bot.cfg.server}")
logging.debug("quit from %s", bot.cfg.server)
bot.state.nrerror += 1

@@ -596,0 +607,0 @@ bot.state.error = evt.txt

+25
-20
rssbot/modules/rss.py

@@ -10,2 +10,3 @@ # This file is placed in the Public Domain.

import http.client
import logging
import os

@@ -25,7 +26,9 @@ import re

from rssbot.methods import elapsed, fmt, rlog, spl
from rssbot.handler import Fleet
from rssbot.objects import Object, update
from rssbot.persist import find, fntime, getpath, last, write
from rssbot.runtime import Repeater, launch
from ..caching import find, last, write
from ..clients import Fleet
from ..methods import fmt
from ..objects import Object, update
from ..threads import Repeater, launch
from ..utility import elapsed, fntime, spl
from ..workdir import getpath

@@ -37,3 +40,5 @@

if fetcher.seenfn:
rlog("warn", f"rss since {elapsed(time.time()-fntime(fetcher.seenfn))}")
logging.warning("since %s", elapsed(time.time()-fntime(fetcher.seenfn)))
else:
logging.warning("since %s", time.ctime(time.time()))
return fetcher

@@ -292,3 +297,3 @@

except (http.client.HTTPException, ValueError, HTTPError, URLError) as ex:
rlog("error", f"{url} {ex}")
logging.error("%s %s", url, ex)
errors[url] = time.time()

@@ -364,3 +369,3 @@ return result

write(feed, fnm)
event.done()
event.reply("ok")

@@ -392,9 +397,9 @@

return
with open(fnm, "r", encoding="utf-8") as file:
txt = file.read()
prs = OPML()
nrs = 0
nrskip = 0
insertid = shortid()
with importlock:
with open(fnm, "r", encoding="utf-8") as file:
txt = file.read()
prs = OPML()
nrs = 0
nrskip = 0
insertid = shortid()
for obj in prs.parse(txt, "outline", "name,display_list,xmlUrl"):

@@ -434,3 +439,3 @@ url = obj.xmlUrl

write(feed, fnm)
event.done()
event.reply("ok")

@@ -449,4 +454,4 @@

feed.__deleted__ = True
write(feed)
event.done()
write(feed, fnm)
event.reply("ok")
break

@@ -459,3 +464,3 @@

return
for fnm, fed in find("rss", deleted=True):
for fnm, fed in find("rss", removed=True):
feed = Rss()

@@ -468,3 +473,3 @@ update(feed, fed)

write(feed, fnm)
event.done()
event.reply("ok")

@@ -494,3 +499,3 @@

write(feed)
event.done()
event.reply("ok")

@@ -497,0 +502,0 @@ 

+3
-59
rssbot/objects.py
# This file is placed in the Public Domain.
"a clean namespace"
"clean namespace"
import json
class Object:

@@ -32,3 +29,3 @@

update(obj, val)
elif isinstance(val, Object):
else:
update(obj, vars(val))

@@ -57,2 +54,3 @@ if kwargs:

def values(obj):

@@ -64,52 +62,2 @@ if isinstance(obj, dict):

"decoder/encoder"
class Encoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, dict):
return o.items()
if issubclass(type(o), Object):
return vars(o)
if isinstance(o, list):
return iter(o)
try:
return json.JSONEncoder.default(self, o)
except TypeError:
try:
return vars(o)
except TypeError:
return repr(o)
def dump(obj, fp, *args, **kw):
kw["cls"] = Encoder
json.dump(obj, fp, *args, **kw)
def dumps(obj, *args, **kw):
kw["cls"] = Encoder
return json.dumps(obj, *args, **kw)
def hook(objdict):
obj = Object()
construct(obj, objdict)
return obj
def load(fp, *args, **kw):
kw["object_hook"] = hook
return json.load(fp, *args, **kw)
def loads(s, *args, **kw):
kw["object_hook"] = hook
return json.loads(s, *args, **kw)
"interface"
def __dir__():

@@ -119,10 +67,6 @@ return (

'construct',
'dump',
'dumps',
'items',
'keys',
'load',
'loads',
'update',
'values'
)
-24
rssbot/modules/tbl.py
# This file is placed in the Public Domain.
NAMES = {
"cfg": "irc",
"dpl": "rss",
"exp": "rss",
"imp": "rss",
"mre": "irc",
"nme": "rss",
"pwd": "irc",
"rem": "rss",
"res": "rss",
"rss": "rss",
"syn": "rss"
}
MD5 = {
"irc": "fc8ddf8baeadfe46b1bed8bf47efecab",
"rss": "b60c574764b55d7c766cd56122946904",
"tbl": "d41d8cd98f00b204e9800998ecf8427e",
}
-215
rssbot/persist.py
# This file is placed in the Public Domain.
"persistence"
import datetime
import json
import os
import pathlib
import threading
import time
from .methods import fqn, j, search
from .objects import Object, dump, load, update
lock = threading.RLock()
class Cache:
objs = {}
@staticmethod
def add(path, obj):
Cache.objs[path] = obj
@staticmethod
def get(path):
return Cache.objs.get(path, None)
@staticmethod
def update(path, obj):
if not obj:
return
if path in Cache.objs:
update(Cache.objs[path], obj)
else:
Cache.add(path, obj)
def cdir(path):
pth = pathlib.Path(path)
pth.parent.mkdir(parents=True, exist_ok=True)
def read(obj, path):
with lock:
with open(path, "r", encoding="utf-8") as fpt:
try:
update(obj, load(fpt))
except json.decoder.JSONDecodeError as ex:
ex.add_note(path)
raise ex
def write(obj, path=None):
with lock:
if path is None:
path = getpath(obj)
cdir(path)
with open(path, "w", encoding="utf-8") as fpt:
dump(obj, fpt, indent=4)
Cache.update(path, obj)
return path
"paths"
class Workdir:
name = __file__.rsplit(os.sep, maxsplit=2)[-2]
wdr = os.path.expanduser(f"~/.{name}")
def getpath(obj):
return store(ident(obj))
def ident(obj):
return j(fqn(obj), *str(datetime.datetime.now()).split())
def long(name):
split = name.split(".")[-1].lower()
res = name
for names in types():
if split == names.split(".")[-1].lower():
res = names
break
return res
def moddir():
return j(Workdir.wdr, "mods")
def pidname(name):
return j(Workdir.wdr, f"{name}.pid")
def setwd(name, path=""):
path = path or os.path.expanduser(f"~/.{name}")
Workdir.wdr = path
skel()
def skel():
if os.path.exists(store()):
return
pth = pathlib.Path(store())
pth.mkdir(parents=True, exist_ok=True)
pth = pathlib.Path(moddir())
pth.mkdir(parents=True, exist_ok=True)
return str(pth)
def store(pth=""):
return j(Workdir.wdr, "store", pth)
def strip(pth, nmr=2):
return j(pth.split(os.sep)[-nmr:])
def types():
skel()
return os.listdir(store())
def wdr(pth):
return j(Workdir.wdr, pth)
"find"
def find(clz, selector=None, deleted=False, matching=False):
clz = long(clz)
if selector is None:
selector = {}
for pth in fns(clz):
obj = Cache.get(pth)
if not obj:
obj = Object()
read(obj, pth)
Cache.add(pth, obj)
if not deleted and isdeleted(obj):
continue
if selector and not search(obj, selector, matching):
continue
yield pth, obj
def fns(clz):
pth = store(clz)
for rootdir, dirs, _files in os.walk(pth, topdown=False):
for dname in dirs:
ddd = j(rootdir, dname)
for fll in os.listdir(ddd):
yield j(ddd, fll)
def fntime(daystr):
datestr = " ".join(daystr.split(os.sep)[-2:])
datestr = datestr.replace("_", " ")
if "." in datestr:
datestr, rest = datestr.rsplit(".", 1)
else:
rest = ""
timed = time.mktime(time.strptime(datestr, "%Y-%m-%d %H:%M:%S"))
if rest:
timed += float("." + rest)
return float(timed)
def isdeleted(obj):
return "__deleted__" in dir(obj) and obj.__deleted__
def last(obj, selector=None):
if selector is None:
selector = {}
result = sorted(find(fqn(obj), selector), key=lambda x: fntime(x[0]))
res = ""
if result:
inp = result[-1]
update(obj, inp[-1])
res = inp[0]
return res
"interface"
def __dir__():
return (
'Cache',
'Workdir',
'cdir',
'find',
'fntime',
'last',
'long',
'pidname',
'read',
'setwd',
'store',
'strip',
'types',
'write'
)
-132
rssbot/runtime.py
# This file is placed in the Public Domain.
"runtime"
import logging
import queue
import threading
import time
import _thread
class Thread(threading.Thread):
def __init__(self, func, *args, daemon=True, **kwargs):
super().__init__(None, self.run, None, (), daemon=daemon)
self.name = kwargs.get("name", name(func))
self.queue = queue.Queue()
self.result = None
self.starttime = time.time()
self.stopped = threading.Event()
self.queue.put((func, args))
def __iter__(self):
return self
def __next__(self):
yield from dir(self)
def run(self):
func, args = self.queue.get()
try:
self.result = func(*args)
except (KeyboardInterrupt, EOFError):
_thread.interrupt_main()
except Exception as ex:
logging.exception(ex)
_thread.interrupt_main()
def join(self, timeout=None):
try:
super().join(timeout)
return self.result
except (KeyboardInterrupt, EOFError):
_thread.interrupt_main()
"timer/repeater"
class Timy(threading.Timer):
def __init__(self, sleep, func, *args, **kwargs):
super().__init__(sleep, func)
self.name = kwargs.get("name", name(func))
self.sleep = sleep
self.state = {}
self.state["latest"] = time.time()
self.state["starttime"] = time.time()
self.starttime = time.time()
class Timed:
def __init__(self, sleep, func, *args, thrname="", **kwargs):
self.args = args
self.func = func
self.kwargs = kwargs
self.sleep = sleep
self.name = thrname or kwargs.get("name", name(func))
self.target = time.time() + self.sleep
self.timer = None
def run(self):
self.timer.latest = time.time()
self.func(*self.args)
def start(self):
self.kwargs["name"] = self.name
timer = Timy(self.sleep, self.run, *self.args, **self.kwargs)
timer.start()
self.timer = timer
def stop(self):
if self.timer:
self.timer.cancel()
class Repeater(Timed):
def run(self):
launch(self.start)
super().run()
"utilities"
def launch(func, *args, **kwargs):
thread = Thread(func, *args, **kwargs)
thread.start()
return thread
def name(obj):
typ = type(obj)
if "__builtins__" in dir(typ):
return obj.__name__
if "__self__" in dir(obj):
return f"{obj.__self__.__class__.__name__}.{obj.__name__}"
if "__class__" in dir(obj) and "__name__" in dir(obj):
return f"{obj.__class__.__name__}.{obj.__name__}"
if "__class__" in dir(obj):
return f"{obj.__class__.__module__}.{obj.__class__.__name__}"
if "__name__" in dir(obj):
return f"{obj.__class__.__name__}.{obj.__name__}"
return ""
"interface"
def __dir__():
return (
'Pool',
'Repeater',
'Thread',
'Timed',
'launch',
'name'
)
bin/rssbot

Sorry, the diff of this file is not supported yet