add logger objects to extractors
This commit is contained in:
@@ -21,6 +21,7 @@ if sys.hexversion < 0x3030000:
|
|||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import logging
|
||||||
import json
|
import json
|
||||||
from . import config, extractor, job, exception
|
from . import config, extractor, job, exception
|
||||||
from .version import __version__
|
from .version import __version__
|
||||||
@@ -111,6 +112,17 @@ def parse_option(opt):
|
|||||||
print("Invalid 'key=value' pair:", opt, file=sys.stderr)
|
print("Invalid 'key=value' pair:", opt, file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def initialize_logging():
|
||||||
|
logging.basicConfig(
|
||||||
|
format="[%(name)s][%(levelname)s] %(message)s",
|
||||||
|
level=logging.INFO
|
||||||
|
)
|
||||||
|
# convert levelnames to lowercase
|
||||||
|
for level in (10, 20, 30, 40, 50):
|
||||||
|
name = logging.getLevelName(level)
|
||||||
|
logging.addLevelName(level, name.lower())
|
||||||
|
|
||||||
|
|
||||||
def sanatize_input(file):
|
def sanatize_input(file):
|
||||||
for line in file:
|
for line in file:
|
||||||
line = line.strip()
|
line = line.strip()
|
||||||
@@ -120,6 +132,7 @@ def sanatize_input(file):
|
|||||||
|
|
||||||
def main():
|
def main():
|
||||||
try:
|
try:
|
||||||
|
initialize_logging()
|
||||||
config.load()
|
config.load()
|
||||||
parser = build_cmdline_parser()
|
parser = build_cmdline_parser()
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
@@ -183,7 +196,6 @@ def main():
|
|||||||
print("No suitable extractor found for URL '", url, "'",
|
print("No suitable extractor found for URL '", url, "'",
|
||||||
sep="", file=sys.stderr)
|
sep="", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
print("\nKeyboardInterrupt", file=sys.stderr)
|
print("\nKeyboardInterrupt", file=sys.stderr)
|
||||||
except BrokenPipeError:
|
except BrokenPipeError:
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
import queue
|
import queue
|
||||||
|
import logging
|
||||||
import requests
|
import requests
|
||||||
import threading
|
import threading
|
||||||
from .message import Message
|
from .message import Message
|
||||||
@@ -26,6 +27,7 @@ class Extractor():
|
|||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.session = requests.Session()
|
self.session = requests.Session()
|
||||||
|
self.log = logging.getLogger(self.category)
|
||||||
|
|
||||||
def __iter__(self):
|
def __iter__(self):
|
||||||
return self.items()
|
return self.items()
|
||||||
|
|||||||
Reference in New Issue
Block a user