chore: Run pyupgrade --py38-plus **/*.py

scrapyd/_deprecate.py

@@ -21,7 +21,7 @@ def __init__(cls, name, bases, clsdict):
                    ['is', 'are'][min(2, len(offending_classes))-1]),
                 ScrapydDeprecationWarning,
             )
-        super(WarningMeta, cls).__init__(name, bases, clsdict)
+        super().__init__(name, bases, clsdict)
 
 
 def deprecate_class(cls):

scrapyd/basicauth.py

@@ -7,7 +7,7 @@
 
 
 @implementer(IRealm)
-class PublicHTMLRealm(object):
+class PublicHTMLRealm:
 
     def __init__(self, resource):
         self.resource = resource
@@ -19,7 +19,7 @@ def requestAvatar(self, avatarId, mind, *interfaces):
 
 
 @implementer(ICredentialsChecker)
-class StringCredentialsChecker(object):
+class StringCredentialsChecker:
     credentialInterfaces = (credentials.IUsernamePassword,)
 
     def __init__(self, username, password):

scrapyd/config.py

@@ -1,13 +1,12 @@
 import glob
-import io
 from configparser import ConfigParser, NoOptionError, NoSectionError
 from os.path import expanduser
 from pkgutil import get_data
 
 from scrapy.utils.conf import closest_scrapy_cfg
 
 
-class Config(object):
+class Config:
     """A ConfigParser wrapper to support defaults when calling instance
     methods, and also tied to a single section"""
 
@@ -22,9 +21,9 @@ def __init__(self, values=None, extra_sources=()):
             sources.extend(extra_sources)
             for fname in sources:
                 try:
-                    with io.open(fname) as fp:
+                    with open(fname) as fp:
                         self.cp.read_file(fp)
-                except (IOError, OSError):
+                except OSError:
                     pass
         else:
             self.cp = ConfigParser(values)

scrapyd/eggstorage.py

@@ -11,7 +11,7 @@
 
 
 @implementer(IEggStorage)
-class FilesystemEggStorage(object):
+class FilesystemEggStorage:
 
     def __init__(self, config):
         self.basedir = config.get('eggs_dir', 'eggs')

scrapyd/environ.py

@@ -10,7 +10,7 @@
 
 
 @implementer(IEnvironment)
-class Environment(object):
+class Environment:
 
     def __init__(self, config, initenv=os.environ):
         self.dbs_dir = config.get('dbs_dir', 'dbs')

scrapyd/jobstorage.py

@@ -15,7 +15,7 @@ def job_items_url(job):
     return f"/items/{job.project}/{job.spider}/{job.job}.jl"
 
 
-class Job(object):
+class Job:
     def __init__(self, project, spider, job=None, start_time=None, end_time=None):
         self.project = project
         self.spider = spider
@@ -25,7 +25,7 @@ def __init__(self, project, spider, job=None, start_time=None, end_time=None):
 
 
 @implementer(IJobStorage)
-class MemoryJobStorage(object):
+class MemoryJobStorage:
 
     def __init__(self, config):
         self.jobs = []
@@ -42,12 +42,11 @@ def __len__(self):
         return len(self.jobs)
 
     def __iter__(self):
-        for j in self.jobs:
-            yield j
+        yield from self.jobs
 
 
 @implementer(IJobStorage)
-class SqliteJobStorage(object):
+class SqliteJobStorage:
 
     def __init__(self, config):
         self.jstorage = SqliteFinishedJobs(sqlite_connection_string(config, 'jobs'), "finished_jobs")

scrapyd/poller.py

@@ -6,7 +6,7 @@
 
 
 @implementer(IPoller)
-class QueuePoller(object):
+class QueuePoller:
 
     def __init__(self, config):
         self.config = config

scrapyd/scheduler.py

@@ -5,7 +5,7 @@
 
 
 @implementer(ISpiderScheduler)
-class SpiderScheduler(object):
+class SpiderScheduler:
 
     def __init__(self, config):
         self.config = config

scrapyd/spiderqueue.py

@@ -6,7 +6,7 @@
 
 
 @implementer(ISpiderQueue)
-class SqliteSpiderQueue(object):
+class SqliteSpiderQueue:
 
     def __init__(self, config, project, table='spider_queue'):
         self.q = JsonSqlitePriorityQueue(sqlite_connection_string(config, project), table)

scrapyd/sqlite.py

@@ -5,7 +5,7 @@
 try:
     from collections.abc import MutableMapping
 except ImportError:
-    from collections import MutableMapping
+    from collections.abc import MutableMapping
 
 
 class JsonSqliteDict(MutableMapping):
@@ -44,8 +44,7 @@ def __len__(self):
         return self.conn.execute(sql).fetchone()[0]
 
     def __iter__(self):
-        for key in self.iterkeys():
-            yield key
+        yield from self.iterkeys()
 
     def iterkeys(self):
         sql = "SELECT key FROM %s" % self.table
@@ -75,7 +74,7 @@ def decode(self, obj):
         return json.loads(bytes(obj).decode('ascii'))
 
 
-class JsonSqlitePriorityQueue(object):
+class JsonSqlitePriorityQueue:
     """SQLite priority queue. It relies on SQLite concurrency support for
     providing atomic inter-process operations.
     """
@@ -141,7 +140,7 @@ def decode(self, text):
         return json.loads(bytes(text).decode('ascii'))
 
 
-class SqliteFinishedJobs(object):
+class SqliteFinishedJobs:
     """SQLite finished jobs. """
 
     def __init__(self, database=None, table="finished_jobs"):
@@ -167,7 +166,7 @@ def clear(self, finished_to_keep=None):
                 return  # nothing to delete
             w = "WHERE id <= " \
                 "(SELECT max(id) FROM (SELECT id FROM %s ORDER BY end_time LIMIT %d))" % (self.table, limit)
-        sql = "DELETE FROM %s %s" % (self.table, w)
+        sql = "DELETE FROM {} {}".format(self.table, w)
         self.conn.execute(sql)
         self.conn.commit()
 

scrapyd/utils.py

@@ -118,10 +118,10 @@ def get_crawl_args(message):
     settings = msg.pop('settings', {})
     for k, v in native_stringify_dict(msg, keys_only=False).items():
         args += ['-a']
-        args += ['%s=%s' % (k, v)]
+        args += ['{}={}'.format(k, v)]
     for k, v in native_stringify_dict(settings, keys_only=False).items():
         args += ['-s']
-        args += ['%s=%s' % (k, v)]
+        args += ['{}={}'.format(k, v)]
     return args
 
 

tests/test_environ.py

@@ -34,10 +34,10 @@ def test_get_environment_with_eggfile(self):
         self.assertEqual(env['SCRAPYD_SLOT'], '3')
         self.assertEqual(env['SCRAPYD_SPIDER'], 'myspider')
         self.assertEqual(env['SCRAPYD_JOB'], 'ID')
-        self.assert_(env['SCRAPYD_LOG_FILE'].endswith(os.path.join('mybot', 'myspider', 'ID.log')))
+        self.assertTrue(env['SCRAPYD_LOG_FILE'].endswith(os.path.join('mybot', 'myspider', 'ID.log')))
         if env.get('SCRAPYD_FEED_URI'):  # Not compulsory
-            self.assert_(env['SCRAPYD_FEED_URI'].startswith('file://{}'.format(os.getcwd())))
-            self.assert_(env['SCRAPYD_FEED_URI'].endswith(os.path.join('mybot', 'myspider', 'ID.jl')))
+            self.assertTrue(env['SCRAPYD_FEED_URI'].startswith(f'file://{os.getcwd()}'))
+            self.assertTrue(env['SCRAPYD_FEED_URI'].endswith(os.path.join('mybot', 'myspider', 'ID.jl')))
         self.assertNotIn('SCRAPY_SETTINGS_MODULE', env)
 
     def test_get_environment_with_no_items_dir(self):

tests/test_spiderqueue.py

@@ -18,7 +18,7 @@ def setUp(self):
         self.args = {
             'arg1': 'val1',
             'arg2': 2,
-            'arg3': u'\N{SNOWMAN}',
+            'arg3': '\N{SNOWMAN}',
         }
         self.msg = self.args.copy()
         self.msg['name'] = self.name

tests/test_sqlite.py

@@ -56,7 +56,7 @@ class JsonSqlitePriorityQueueTest(unittest.TestCase):
 
     supported_values = [
         "native ascii str",
-        u"\xa3",
+        "\xa3",
         123,
         1.2,
         True,

tests/test_utils.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 from io import BytesIO
 from pkgutil import get_data
@@ -27,14 +26,14 @@ def test_get_crawl_args(self):
 
         self.assertEqual(get_crawl_args(msg), ['lala'])
 
-        msg = {'_project': 'lolo', '_spider': 'lala', 'arg1': u'val1'}
+        msg = {'_project': 'lolo', '_spider': 'lala', 'arg1': 'val1'}
         cargs = get_crawl_args(msg)
 
         self.assertEqual(cargs, ['lala', '-a', 'arg1=val1'])
         self.assertTrue(all(isinstance(x, str) for x in cargs), cargs)
 
     def test_get_crawl_args_with_settings(self):
-        msg = {'_project': 'lolo', '_spider': 'lala', 'arg1': u'val1', 'settings': {'ONE': 'two'}}
+        msg = {'_project': 'lolo', '_spider': 'lala', 'arg1': 'val1', 'settings': {'ONE': 'two'}}
         cargs = get_crawl_args(msg)
 
         self.assertEqual(cargs, ['lala', '-a', 'arg1=val1', '-s', 'ONE=two'])
@@ -110,7 +109,7 @@ def test_get_spider_list_unicode(self):
         self.add_test_version('mybotunicode.egg', 'mybotunicode', 'r1')
         spiders = get_spider_list('mybotunicode', pythonpath=get_pythonpath_scrapyd())
 
-        self.assertEqual(sorted(spiders), [u'araña1', u'araña2'])
+        self.assertEqual(sorted(spiders), ['araña1', 'araña2'])
 
     def test_failed_spider_list(self):
         self.add_test_version('mybot3.egg', 'mybot3', 'r1')