Useful Insights and Workflow with Virtual Environment

This content is from @johnziebro (see https://github.com/montag451/pypi-mirror/issues/14 for the original content)

Virtual Environment New Project Workflow w/ Pyenv*:

Assumes Pyenv installed.

cd ~/Projects
mkdir new_proj
cd new_proj
pyenv local 3.8.12
python -m venv venv
source venv/bin/activate
pip config set global.trusted-host "localhost 0.0.0.0"

*note global.trusted only affects the venv not base os

Download one or any number of modules as binaries: pypi-mirror download -d downloads -b pip requests flask

Generate a pypi repository from downloads: pypi-mirror create -d downloads -m simple

Use python's webserver to serve locally: python3 -m http.server 8000

Pip install from the repository untrustingly: pip install -i http://0.0.0.0:8000/simple requests --trusted-host 0.0.0.0

Upgrade pip from local trusted mirror (if pip is mirrored) pip install -i http://localhost:8000/simple --upgrade pip

Install multiple modules from requirements.txt from local mirror trustingly

echo -e "requests\nflask" > requirements.txt
pip install -i http://localhost:8000/simple -r requirements.txt

Install specific modules from local mirror trustingly pip install -i http://localhost:8000/simple requests flask

Install multiple files from requirements.txt from local mirror untrustingly

echo -e "requests\nflask" > requirements.txt
pip install -i http://localhost:8000/simple -r requirements.txt --trusted-host localhost

Update all installed modules pypi-mirror list -d /path/to/my/download/dir --name-only | xargs pypi-mirror download -d /path/to/my/download/dir

Additional notes on pip's config inside venv:

pip config set global.trusted-host "0.0.0.0 localhost"
pip config list
nano $VIRTUAL_ENV/pip.conf

Simple Python server script. Place in directory above 'simple' directory.

#!/usr/bin/env python3
#server.py
"""
Very simple HTTP server in python for logging usage of
pypi_mirror to the console and file; log.txt.
Usage::
    python server.py [<port>]
"""
import pypi_mirror
import logging
import textwrap
import http.server
import socketserver
from sys import argv
from pathlib import Path
from importlib.metadata import version as ver


class ServerHandler(http.server.SimpleHTTPRequestHandler):
    """ Inherits from SimpleHTTPRequestHandler to provide logging.
    Serves a subdirectory, simple/, in accordance with PEP 503.
    https://www.python.org/dev/peps/pep-0503/ """

    buffer = 1
    log_file = open('log.txt', 'a', buffer)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, directory=DIRECTORY, **kwargs)

    def log_message(self, format, *args):
        stamp = self.log_date_time_string()
        from_ip = self.client_address[0]
        command = self.command
        path = self.path
        host = self.headers._headers[0][1]
        user_agent = self.headers._headers[1][1].split(" ")[0]
        req_ver = self.request_version
        info = f' {from_ip} - [{stamp}] "{user_agent} {command} {req_ver} {host}{path}"'
        logging.info(info)
        self.log_file.write(f"{info}\n")


def startup():
    """ Provides startup messaging. """
    messages = [
        f"\n\nPython-pypi-mirror v{VERSION} Simple Server",
        "-"*75,
        f"          Artifacts: {ARTIFACTS}",
        f"            Serving: {SERVE_DIR}",
        f"               Port: {PORT}",
        f"             Source: https://github.com/montag451/pypi-mirror",
        "\nNOTES",
        "-"*75,
        textwrap.dedent("""\
        pypi-mirror is a small script to generate a partial PyPI mirror. It
        relies on pip to download a package and its dependencies. Most of the
        time you don't need a full PyPI mirror but only a mirror that contains
        the packages you use. If you want a full PyPI mirror you should look
        at bandersnatch. It is recommended to run pypi_mirror in a virtual
        environment. The following commands assume running in a virtual
        environment with Pyenv installed."""),
        "             Mirror: pypi-mirror download -d downloads -b pip requests flask",
        "          Generate : pypi-mirror create -d downloads -m simple",
        '         Trust Host: pip config set global.trusted-host "localhost 0.0.0.0"',
        f"    Trusted Install: pip install -i http://localhost:{PORT} requests",
        f"  Untrusted Install: pip install -i http://localhost:{PORT} requests --trusted-host localhost\n",
    ]
    intro = "\n".join(messages)
    logging.info(intro)
    logging.info(' Starting httpd...\n')


def run(PORT:int):
    """ Starts and stops the simple server on a specific port. """
    with socketserver.TCPServer(("", PORT), ServerHandler) as httpd:
        startup()
        try:
            httpd.serve_forever()
        except KeyboardInterrupt:
            pass
        logging.info(' Stopping httpd...\n')
        httpd.RequestHandlerClass.log_file.close()
        httpd.server_close()


if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    SERVE_DIR = Path().resolve() / 'simple'
    ARTIFACTS = sum(1 for x in SERVE_DIR.glob('*')) - 1 # account for index.html
    VERSION = ver('python-pypi-mirror')
    DIRECTORY = "simple"
    PORT = 8080
    if len(argv) == 2:
        PORT=int(argv[1])
        run(PORT=PORT)
    else:
        run(PORT=PORT)