old htb folders
This commit is contained in:
2023-08-29 21:53:22 +02:00
parent 62ab804867
commit 82b0759f1e
21891 changed files with 6277643 additions and 0 deletions

View File

@@ -0,0 +1,119 @@
r"""
Plumbum Shell Combinators
-------------------------
A wrist-handy library for writing shell-like scripts in Python, that can serve
as a ``Popen`` replacement, and much more::
>>> from plumbum.cmd import ls, grep, wc, cat
>>> ls()
'build.py\ndist\ndocs\nLICENSE\nplumbum\nREADME.rst\nsetup.py\ntests\ntodo.txt\n'
>>> chain = ls["-a"] | grep["-v", "py"] | wc["-l"]
>>> print(chain)
/bin/ls -a | /bin/grep -v py | /usr/bin/wc -l
>>> chain()
'12\n'
>>> ((ls["-a"] | grep["-v", "py"]) > "/tmp/foo.txt")()
''
>>> ((cat < "/tmp/foo.txt") | wc["-l"])()
'12\n'
>>> from plumbum import local, FG, BG
>>> with local.cwd("/tmp"):
... (ls | wc["-l"]) & FG
...
13 # printed directly to the interpreter's stdout
>>> (ls | wc["-l"]) & BG
<Future ['/usr/bin/wc', '-l'] (running)>
>>> f = _
>>> f.stdout # will wait for the process to terminate
'9\n'
Plumbum includes local/remote path abstraction, working directory and environment
manipulation, process execution, remote process execution over SSH, tunneling,
SCP-based upload/download, and a {arg|opt}parse replacement for the easy creation
of command-line interface (CLI) programs.
See https://plumbum.readthedocs.io for full details
"""
import sys
# Avoids a circular import error later
import plumbum.path # noqa: F401
from plumbum.commands import (
BG,
ERROUT,
FG,
NOHUP,
RETCODE,
TEE,
TF,
CommandNotFound,
ProcessExecutionError,
ProcessLineTimedOut,
ProcessTimedOut,
)
from plumbum.machines import BaseRemoteMachine, PuttyMachine, SshMachine, local
from plumbum.path import LocalPath, Path, RemotePath
from plumbum.version import version
__author__ = "Tomer Filiba (tomerfiliba@gmail.com)"
__version__ = version
__all__ = (
"BG",
"ERROUT",
"FG",
"NOHUP",
"RETCODE",
"TEE",
"TF",
"CommandNotFound",
"ProcessExecutionError",
"ProcessLineTimedOut",
"ProcessTimedOut",
"BaseRemoteMachine",
"PuttyMachine",
"SshMachine",
"local",
"LocalPath",
"Path",
"RemotePath",
"__author__",
"__version__",
"cmd",
)
# ===================================================================================================
# Module hack: ``from plumbum.cmd import ls``
# Can be replaced by a real module with __getattr__ after Python 3.6 is dropped
# ===================================================================================================
if sys.version_info < (3, 7):
from types import ModuleType
from typing import List
class LocalModule(ModuleType):
"""The module-hack that allows us to use ``from plumbum.cmd import some_program``"""
__all__ = () # to make help() happy
__package__ = __name__
def __getattr__(self, name):
try:
return local[name]
except CommandNotFound:
raise AttributeError(name) from None
__path__: List[str] = []
__file__ = __file__
cmd = LocalModule(__name__ + ".cmd", LocalModule.__doc__)
sys.modules[cmd.__name__] = cmd
else:
from . import cmd
def __dir__():
"Support nice tab completion"
return __all__

View File

@@ -0,0 +1,24 @@
import os
import platform
import sys
import pytest # type: ignore[import]
skip_without_chown = pytest.mark.skipif(
not hasattr(os, "chown"), reason="os.chown not supported"
)
skip_without_tty = pytest.mark.skipif(not sys.stdin.isatty(), reason="Not a TTY")
skip_on_windows = pytest.mark.skipif(
sys.platform == "win32", reason="Windows not supported for this test (yet)"
)
xfail_on_windows = pytest.mark.xfail(
sys.platform == "win32", reason="Windows not supported for this test (yet)"
)
xfail_on_pypy = pytest.mark.xfail(
platform.python_implementation() == "PyPy",
reason="PyPy is currently not working on this test!",
)

View File

@@ -0,0 +1,38 @@
from .application import Application
from .config import Config, ConfigINI
from .switches import (
CSV,
CountOf,
ExistingDirectory,
ExistingFile,
Flag,
NonexistentPath,
Predicate,
Range,
Set,
SwitchAttr,
SwitchError,
autoswitch,
positional,
switch,
)
__all__ = (
"Application",
"Config",
"ConfigINI",
"CSV",
"CountOf",
"ExistingDirectory",
"ExistingFile",
"Flag",
"NonexistentPath",
"Predicate",
"Range",
"Set",
"SwitchAttr",
"SwitchError",
"autoswitch",
"positional",
"switch",
)

View File

@@ -0,0 +1,118 @@
import contextlib
from abc import ABC, abstractmethod
from configparser import ConfigParser, NoOptionError, NoSectionError
from plumbum import local
class ConfigBase(ABC):
"""Base class for Config parsers.
:param filename: The file to use
The ``with`` statement can be used to automatically try to read on entering and write if changed on exiting. Otherwise, use ``.read`` and ``.write`` as needed. Set and get the options using ``[]`` syntax.
Usage:
with Config("~/.myprog_rc") as conf:
value = conf.get("option", "default")
value2 = conf["option"] # shortcut for default=None
"""
__slots__ = "filename changed".split()
def __init__(self, filename):
self.filename = local.path(filename)
self.changed = False
def __enter__(self):
with contextlib.suppress(FileNotFoundError):
self.read()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self.changed:
self.write()
@abstractmethod
def read(self):
"""Read in the linked file"""
@abstractmethod
def write(self):
"""Write out the linked file"""
self.changed = False
@abstractmethod
def _get(self, option):
"""Internal get function for subclasses"""
@abstractmethod
def _set(self, option, value):
"""Internal set function for subclasses. Must return the value that was set."""
def get(self, option, default=None):
"Get an item from the store, returns default if fails"
try:
return self._get(option)
except KeyError:
self.changed = True
return self._set(option, default)
def set(self, option, value):
"""Set an item, mark this object as changed"""
self.changed = True
self._set(option, value)
def __getitem__(self, option):
return self._get(option)
def __setitem__(self, option, value):
return self.set(option, value)
class ConfigINI(ConfigBase):
DEFAULT_SECTION = "DEFAULT"
slots = "parser".split()
def __init__(self, filename):
super().__init__(filename)
self.parser = ConfigParser()
def read(self):
self.parser.read(self.filename)
super().read()
def write(self):
with open(self.filename, "w", encoding="utf-8") as f:
self.parser.write(f)
super().write()
@classmethod
def _sec_opt(cls, option):
if "." not in option:
sec = cls.DEFAULT_SECTION
else:
sec, option = option.split(".", 1)
return sec, option
def _get(self, option):
sec, option = self._sec_opt(option)
try:
return self.parser.get(sec, option)
except (NoSectionError, NoOptionError):
raise KeyError(f"{sec}:{option}") from None
def _set(self, option, value):
sec, option = self._sec_opt(option)
try:
self.parser.set(sec, option, str(value))
except NoSectionError:
self.parser.add_section(sec)
self.parser.set(sec, option, str(value))
return str(value)
Config = ConfigINI

View File

@@ -0,0 +1,53 @@
import locale
# High performance method for English (no translation needed)
loc = locale.getlocale()[0]
if loc is None or loc.startswith("en"):
class NullTranslation:
def gettext(self, str1: str) -> str: # pylint: disable=no-self-use
return str1
def ngettext(self, str1, strN, n): # pylint: disable=no-self-use
if n == 1:
return str1.replace("{0}", str(n))
return strN.replace("{0}", str(n))
def get_translation_for(
package_name: str, # pylint: disable=unused-argument
) -> NullTranslation:
return NullTranslation()
else:
import gettext
import os
# If not installed with setuptools, this might not be available
try:
import pkg_resources
except ImportError:
pkg_resources = None # type: ignore[assignment]
local_dir = os.path.basename(__file__)
def get_translation_for(package_name: str) -> gettext.NullTranslations: # type: ignore[misc]
"""Find and return gettext translation for package
(Try to find folder manually if setuptools does not exist)
"""
if "." in package_name:
package_name = ".".join(package_name.split(".")[:-1])
localedir = None
if pkg_resources is None:
mydir = os.path.join(local_dir, "i18n")
else:
mydir = pkg_resources.resource_filename(package_name, "i18n")
for localedir in mydir, None:
localefile = gettext.find(package_name, localedir)
if localefile:
break
return gettext.translation(package_name, localedir=localedir, fallback=True)

View File

@@ -0,0 +1,233 @@
# German Translations for PACKAGE package.
# Deutsche Übersetzung für PACKAGE paket.
# Copyright (C) 2017 THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# Christoph Hasse <christoph.hasse@cern.ch> , 2017.
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2022-10-05 22:39-0400\n"
"PO-Revision-Date: 2017-11-02 15:04+0200\n"
"Last-Translator: Christoph Hasse <christoph.hasse@cern.ch> \n"
"Language: de_DE\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
#: plumbum/cli/application.py:69
#, python-brace-format
msgid "Subcommand({self.name}, {self.subapplication})"
msgstr "Unterbefehl({self.name}, {self.subapplication})"
#: plumbum/cli/application.py:73
msgid "Switches"
msgstr "Optionen"
#: plumbum/cli/application.py:73
msgid "Meta-switches"
msgstr "Meta-optionen"
#: plumbum/cli/application.py:163
#, python-brace-format
msgid "see '{parent} {sub} --help' for more info"
msgstr "siehe '{parent} {sub} --help' für mehr Informationen"
#: plumbum/cli/application.py:220
#, fuzzy
msgid "Sub-command names cannot start with '-'"
msgstr "Unterbefehle können nicht mit '-' beginnen"
#: plumbum/cli/application.py:238
#, fuzzy, python-brace-format
msgid "Switch {name} already defined and is not overridable"
msgstr "Option {name} ist bereits definiert und nicht überschreibbar"
#: plumbum/cli/application.py:343
#, python-brace-format
msgid "Ambiguous partial switch {0}"
msgstr ""
#: plumbum/cli/application.py:348 plumbum/cli/application.py:373
#: plumbum/cli/application.py:389
#, python-brace-format
msgid "Unknown switch {0}"
msgstr "Unbekannte Option {0}"
#: plumbum/cli/application.py:353 plumbum/cli/application.py:362
#: plumbum/cli/application.py:381
#, python-brace-format
msgid "Switch {0} requires an argument"
msgstr "Option {0} benötigt ein Argument"
#: plumbum/cli/application.py:401
#, python-brace-format
msgid "Switch {0} already given"
msgstr "Option {0} bereits gegeben"
#: plumbum/cli/application.py:403
#, python-brace-format
msgid "Switch {0} already given ({1} is equivalent)"
msgstr "Option {0} bereits gegeben({1} ist äquivalent)"
#: plumbum/cli/application.py:451
msgid ""
"Argument of {name} expected to be {argtype}, not {val!r}:\n"
" {ex!r}"
msgstr ""
"Argument von {name} sollte {argtype} sein, nicht {val|1}:\n"
" {ex!r}"
#: plumbum/cli/application.py:470
#, python-brace-format
msgid "Switch {0} is mandatory"
msgstr "Option {0} ist notwendig"
#: plumbum/cli/application.py:490
#, python-brace-format
msgid "Given {0}, the following are missing {1}"
msgstr "Gegeben {0}, werden die folgenden vermisst {1}"
#: plumbum/cli/application.py:498
#, python-brace-format
msgid "Given {0}, the following are invalid {1}"
msgstr "Gegeben {0}, sind die folgenden ungültig {1}"
#: plumbum/cli/application.py:515
#, python-brace-format
msgid "Expected at least {0} positional argument, got {1}"
msgid_plural "Expected at least {0} positional arguments, got {1}"
msgstr[0] "Erwarte mindestens {0} positionelles Argument, erhalte {1}"
msgstr[1] "Erwarte mindestens {0} positionelle Argumente, erhalte {1}"
#: plumbum/cli/application.py:523
#, python-brace-format
msgid "Expected at most {0} positional argument, got {1}"
msgid_plural "Expected at most {0} positional arguments, got {1}"
msgstr[0] "Erwarte höchstens {0} positionelles Argument, erhalte {0}"
msgstr[1] "Erwarte höchstens {0} positionelle Argumente, erhalte {0}"
#: plumbum/cli/application.py:624
#, python-brace-format
msgid "Error: {0}"
msgstr "Fehler: {0}"
#: plumbum/cli/application.py:625 plumbum/cli/application.py:711
#: plumbum/cli/application.py:716
msgid "------"
msgstr "------"
#: plumbum/cli/application.py:694
#, python-brace-format
msgid "Switch {0} must be a sequence (iterable)"
msgstr "Option {0} muss eine Sequenz sein (iterierbar)"
#: plumbum/cli/application.py:699
#, python-brace-format
msgid "Switch {0} is a boolean flag"
msgstr "Option {0} ist ein boolescher Wert"
#: plumbum/cli/application.py:710
#, python-brace-format
msgid "Unknown sub-command '{0}'"
msgstr "Unbekannter Unterbefehl '{0}'"
#: plumbum/cli/application.py:715
msgid "No sub-command given"
msgstr "Kein Unterbefehl gegeben"
#: plumbum/cli/application.py:721
msgid "main() not implemented"
msgstr "main() nicht implementiert"
#: plumbum/cli/application.py:734
#, fuzzy
msgid "Prints help messages of all sub-commands and quits"
msgstr "Druckt die Hilfetexte aller Unterbefehle und terminiert"
#: plumbum/cli/application.py:754
msgid "Prints this help message and quits"
msgstr "Druckt den Hilfetext und terminiert"
#: plumbum/cli/application.py:877
msgid "Usage:"
msgstr "Gebrauch:"
#: plumbum/cli/application.py:883
#, python-brace-format
msgid " {progname} [SWITCHES] [SUBCOMMAND [SWITCHES]] {tailargs}\n"
msgstr " {progname} [OPTIONEN] [UNTERBEFEHL [OPTIONEN]] {tailargs}\n"
#: plumbum/cli/application.py:886
#, python-brace-format
msgid " {progname} [SWITCHES] {tailargs}\n"
msgstr " {progname} [OPTIONEN] {tailargs}\n"
#: plumbum/cli/application.py:936
msgid "; may be given multiple times"
msgstr "; kann mehrmals angegeben werden"
#: plumbum/cli/application.py:938
msgid "; required"
msgstr "; benötigt"
#: plumbum/cli/application.py:940
#, python-brace-format
msgid "; requires {0}"
msgstr "; benötigt {0}"
#: plumbum/cli/application.py:947
#, python-brace-format
msgid "; excludes {0}"
msgstr "; schließt {0} aus"
#: plumbum/cli/application.py:966
#, fuzzy
msgid "Sub-commands:"
msgstr "Unterbefehle:"
#: plumbum/cli/application.py:1014
msgid "Prints the program's version and quits"
msgstr "Druckt die Programmversion und terminiert"
#: plumbum/cli/application.py:1019
msgid "(version not set)"
msgstr "(Version nicht gesetzt)"
#: plumbum/cli/switches.py:167 plumbum/cli/switches.py:225
msgid "VALUE"
msgstr "WERT"
#: plumbum/cli/switches.py:238
#, python-brace-format
msgid "; the default is {0}"
msgstr "; der Standard ist {0}"
#: plumbum/cli/switches.py:437
#, python-brace-format
msgid "Not in range [{0:d}..{1:d}]"
msgstr "Nicht im Wertebereich [{0:d}..{1:d}]"
#: plumbum/cli/switches.py:546
#, python-brace-format
msgid "{0} is not a directory"
msgstr "{0} ist kein Ordner"
#: plumbum/cli/switches.py:565
#, python-brace-format
msgid "{0} is not a file"
msgstr "{0} ist keine Datei"
#: plumbum/cli/switches.py:574
#, python-brace-format
msgid "{0} already exists"
msgstr "{0} existiert bereits"
#, python-brace-format
#~ msgid "got unexpected keyword argument(s): {0}"
#~ msgstr "Unerwartete(s) Argument(e) erhalten: {0}"
#, python-brace-format
#~ msgid "Expected one of {0}"
#~ msgstr "Erwartet einen von {0}"

View File

@@ -0,0 +1,233 @@
# French Translations for PACKAGE package.
# Traduction francaise du paquet PACKAGE.
# Copyright (C) 2017 THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# Joel Closier <joel.closier@cern.ch>, 2017.
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2022-10-05 22:39-0400\n"
"PO-Revision-Date: 2017-10-14 15:04+0200\n"
"Last-Translator: Joel Closier <joel.closier@cern.ch>\n"
"Language: fr_FR\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
#: plumbum/cli/application.py:69
#, python-brace-format
msgid "Subcommand({self.name}, {self.subapplication})"
msgstr "Sous-commande({self.name}, {self.subapplication})"
#: plumbum/cli/application.py:73
msgid "Switches"
msgstr "Options"
#: plumbum/cli/application.py:73
msgid "Meta-switches"
msgstr "Meta-options"
#: plumbum/cli/application.py:163
#, python-brace-format
msgid "see '{parent} {sub} --help' for more info"
msgstr "voir '{parent} {sub} --help' pour plus d'information"
#: plumbum/cli/application.py:220
#, fuzzy
msgid "Sub-command names cannot start with '-'"
msgstr "le nom des Sous-commandes ne peut pas commencer avec '-' "
#: plumbum/cli/application.py:238
#, fuzzy, python-brace-format
msgid "Switch {name} already defined and is not overridable"
msgstr "Option {name} est déjà définie et ne peut pas être sur-écrite"
#: plumbum/cli/application.py:343
#, python-brace-format
msgid "Ambiguous partial switch {0}"
msgstr ""
#: plumbum/cli/application.py:348 plumbum/cli/application.py:373
#: plumbum/cli/application.py:389
#, python-brace-format
msgid "Unknown switch {0}"
msgstr "Option inconnue {0}"
#: plumbum/cli/application.py:353 plumbum/cli/application.py:362
#: plumbum/cli/application.py:381
#, python-brace-format
msgid "Switch {0} requires an argument"
msgstr "Option {0} nécessite un argument"
#: plumbum/cli/application.py:401
#, python-brace-format
msgid "Switch {0} already given"
msgstr "Option {0} déjà donnée"
#: plumbum/cli/application.py:403
#, python-brace-format
msgid "Switch {0} already given ({1} is equivalent)"
msgstr "Option {0} déjà donnée ({1} est équivalent)"
#: plumbum/cli/application.py:451
msgid ""
"Argument of {name} expected to be {argtype}, not {val!r}:\n"
" {ex!r}"
msgstr ""
"Argument de {name} doit être {argtype} , et non {val!r}:\n"
" {ex!r}"
#: plumbum/cli/application.py:470
#, python-brace-format
msgid "Switch {0} is mandatory"
msgstr "Option {0} obligatoire"
#: plumbum/cli/application.py:490
#, python-brace-format
msgid "Given {0}, the following are missing {1}"
msgstr "Etant donné {0}, ce qui suit est manquant {1}"
#: plumbum/cli/application.py:498
#, python-brace-format
msgid "Given {0}, the following are invalid {1}"
msgstr "Etant donné {0}, ce qui suit est invalide {1}"
#: plumbum/cli/application.py:515
#, python-brace-format
msgid "Expected at least {0} positional argument, got {1}"
msgid_plural "Expected at least {0} positional arguments, got {1}"
msgstr[0] "Au moins {0} argument de position attendu, reçu {0}"
msgstr[1] "Au moins {0} arguments de position, reçu {0}"
#: plumbum/cli/application.py:523
#, python-brace-format
msgid "Expected at most {0} positional argument, got {1}"
msgid_plural "Expected at most {0} positional arguments, got {1}"
msgstr[0] "Au plus {0} argument de position attendu, reçu {0}"
msgstr[1] "Au plus {0} arguments de position, reçu {0}"
#: plumbum/cli/application.py:624
#, python-brace-format
msgid "Error: {0}"
msgstr "Erreur: {0}"
#: plumbum/cli/application.py:625 plumbum/cli/application.py:711
#: plumbum/cli/application.py:716
msgid "------"
msgstr "------"
#: plumbum/cli/application.py:694
#, python-brace-format
msgid "Switch {0} must be a sequence (iterable)"
msgstr "Option {0} doit être une séquence (itérable)"
#: plumbum/cli/application.py:699
#, python-brace-format
msgid "Switch {0} is a boolean flag"
msgstr "Option {0} est un booléen"
#: plumbum/cli/application.py:710
#, python-brace-format
msgid "Unknown sub-command '{0}'"
msgstr "Sous-commande inconnue '{0}'"
#: plumbum/cli/application.py:715
msgid "No sub-command given"
msgstr "Pas de sous-commande donnée"
#: plumbum/cli/application.py:721
msgid "main() not implemented"
msgstr "main() n'est pas implémenté"
#: plumbum/cli/application.py:734
#, fuzzy
msgid "Prints help messages of all sub-commands and quits"
msgstr "Imprime les messages d'aide de toutes les sous-commandes et sort"
#: plumbum/cli/application.py:754
msgid "Prints this help message and quits"
msgstr "Imprime ce message d'aide et sort"
#: plumbum/cli/application.py:877
msgid "Usage:"
msgstr "Utilisation:"
#: plumbum/cli/application.py:883
#, python-brace-format
msgid " {progname} [SWITCHES] [SUBCOMMAND [SWITCHES]] {tailargs}\n"
msgstr " {progname} [OPTIONS] [SOUS_COMMANDE [OPTIONS]] {tailargs}\n"
#: plumbum/cli/application.py:886
#, python-brace-format
msgid " {progname} [SWITCHES] {tailargs}\n"
msgstr " {progname} [OPTIONS] {tailargs}\n"
#: plumbum/cli/application.py:936
msgid "; may be given multiple times"
msgstr "; peut être fourni plusieurs fois"
#: plumbum/cli/application.py:938
msgid "; required"
msgstr "; nécessaire"
#: plumbum/cli/application.py:940
#, python-brace-format
msgid "; requires {0}"
msgstr "; nécessite {0}"
#: plumbum/cli/application.py:947
#, python-brace-format
msgid "; excludes {0}"
msgstr "; exclut {0}"
#: plumbum/cli/application.py:966
#, fuzzy
msgid "Sub-commands:"
msgstr "Sous-Commandes:"
#: plumbum/cli/application.py:1014
msgid "Prints the program's version and quits"
msgstr "Imprime la version du programme et sort"
#: plumbum/cli/application.py:1019
msgid "(version not set)"
msgstr "(version non définie)"
#: plumbum/cli/switches.py:167 plumbum/cli/switches.py:225
msgid "VALUE"
msgstr "VALEUR"
#: plumbum/cli/switches.py:238
#, python-brace-format
msgid "; the default is {0}"
msgstr "; la valeur par défaut est {0}"
#: plumbum/cli/switches.py:437
#, python-brace-format
msgid "Not in range [{0:d}..{1:d}]"
msgstr "Pas dans la chaîne [{0:d}..{1:d}]"
#: plumbum/cli/switches.py:546
#, python-brace-format
msgid "{0} is not a directory"
msgstr "{0} n'est pas un répertoire"
#: plumbum/cli/switches.py:565
#, python-brace-format
msgid "{0} is not a file"
msgstr "{0} n'est pas un fichier"
#: plumbum/cli/switches.py:574
#, python-brace-format
msgid "{0} already exists"
msgstr "{0} existe déjà"
#, python-brace-format
#~ msgid "got unexpected keyword argument(s): {0}"
#~ msgstr "mot-clé inconnu donné comme argument: {0}"
#, python-brace-format
#~ msgid "Expected one of {0}"
#~ msgstr "un des {0} attendu"

View File

@@ -0,0 +1,233 @@
# Dutch Translations for PACKAGE package.
# Nederlandse vertaling voor het PACKAGE pakket.
# Copyright (C) 2017 THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# Roel Aaij <roel.aaij@gmail.com>, 2017.
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2022-10-05 22:39-0400\n"
"PO-Revision-Date: 2017-10-14 15:04+0200\n"
"Last-Translator: Roel Aaij <roel.aaij@gmail.com>\n"
"Language: nl_NL\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
#: plumbum/cli/application.py:69
#, python-brace-format
msgid "Subcommand({self.name}, {self.subapplication})"
msgstr "Subopdracht({self.name}, {self.subapplication})"
#: plumbum/cli/application.py:73
msgid "Switches"
msgstr "Opties"
#: plumbum/cli/application.py:73
msgid "Meta-switches"
msgstr "Meta-opties"
#: plumbum/cli/application.py:163
#, python-brace-format
msgid "see '{parent} {sub} --help' for more info"
msgstr "zie '{parent} {sub} --help' voor meer informatie"
#: plumbum/cli/application.py:220
#, fuzzy
msgid "Sub-command names cannot start with '-'"
msgstr "Namen van subopdrachten mogen niet met '-' beginnen"
#: plumbum/cli/application.py:238
#, fuzzy, python-brace-format
msgid "Switch {name} already defined and is not overridable"
msgstr "Optie {name} is al gedefiniëerd en kan niet worden overschreven"
#: plumbum/cli/application.py:343
#, python-brace-format
msgid "Ambiguous partial switch {0}"
msgstr ""
#: plumbum/cli/application.py:348 plumbum/cli/application.py:373
#: plumbum/cli/application.py:389
#, python-brace-format
msgid "Unknown switch {0}"
msgstr "Onbekende optie {0}"
#: plumbum/cli/application.py:353 plumbum/cli/application.py:362
#: plumbum/cli/application.py:381
#, python-brace-format
msgid "Switch {0} requires an argument"
msgstr "Een argument is vereist bij optie {0}"
#: plumbum/cli/application.py:401
#, python-brace-format
msgid "Switch {0} already given"
msgstr "Optie {0} is al gegeven"
#: plumbum/cli/application.py:403
#, python-brace-format
msgid "Switch {0} already given ({1} is equivalent)"
msgstr "Optie {0} is al gegeven ({1} is equivalent)"
#: plumbum/cli/application.py:451
msgid ""
"Argument of {name} expected to be {argtype}, not {val!r}:\n"
" {ex!r}"
msgstr ""
"Argement van {name} hoort {argtype} te zijn, niet {val|1}:\n"
" {ex!r}"
#: plumbum/cli/application.py:470
#, python-brace-format
msgid "Switch {0} is mandatory"
msgstr "Optie {0} is verplicht"
#: plumbum/cli/application.py:490
#, python-brace-format
msgid "Given {0}, the following are missing {1}"
msgstr "Gegeven {0}, ontbreken de volgenden {1}"
#: plumbum/cli/application.py:498
#, python-brace-format
msgid "Given {0}, the following are invalid {1}"
msgstr "Gegeven {0}, zijn de volgenden ongeldig {1}"
#: plumbum/cli/application.py:515
#, python-brace-format
msgid "Expected at least {0} positional argument, got {1}"
msgid_plural "Expected at least {0} positional arguments, got {1}"
msgstr[0] "Verwachtte ten minste {0} positioneel argument, kreeg {1}"
msgstr[1] "Verwachtte ten minste {0} positionele argumenten, kreeg {1}"
#: plumbum/cli/application.py:523
#, python-brace-format
msgid "Expected at most {0} positional argument, got {1}"
msgid_plural "Expected at most {0} positional arguments, got {1}"
msgstr[0] "Verwachtte hoogstens {0} positioneel argument, kreeg {0}"
msgstr[1] "Verwachtte hoogstens {0} positionele argumenten, kreeg {0}"
#: plumbum/cli/application.py:624
#, python-brace-format
msgid "Error: {0}"
msgstr "Fout: {0}"
#: plumbum/cli/application.py:625 plumbum/cli/application.py:711
#: plumbum/cli/application.py:716
msgid "------"
msgstr "------"
#: plumbum/cli/application.py:694
#, python-brace-format
msgid "Switch {0} must be a sequence (iterable)"
msgstr "Optie {0} moet een reeks zijn (itereerbaar object)"
#: plumbum/cli/application.py:699
#, python-brace-format
msgid "Switch {0} is a boolean flag"
msgstr "Optie {0} geeft een waarheidswaarde weer"
#: plumbum/cli/application.py:710
#, python-brace-format
msgid "Unknown sub-command '{0}'"
msgstr "Onbekend subcommando '{0}'"
#: plumbum/cli/application.py:715
msgid "No sub-command given"
msgstr "Geen subcommando gegeven"
#: plumbum/cli/application.py:721
msgid "main() not implemented"
msgstr "main() niet geïmplementeerd"
#: plumbum/cli/application.py:734
#, fuzzy
msgid "Prints help messages of all sub-commands and quits"
msgstr "Druk hulpberichten van alle subcommando's af en beëindig"
#: plumbum/cli/application.py:754
msgid "Prints this help message and quits"
msgstr "Drukt dit hulpbericht af en beëindig"
#: plumbum/cli/application.py:877
msgid "Usage:"
msgstr "Gebruik:"
#: plumbum/cli/application.py:883
#, python-brace-format
msgid " {progname} [SWITCHES] [SUBCOMMAND [SWITCHES]] {tailargs}\n"
msgstr " {progname} [OPTIES] [SUBCOMMANDO [OPTIES]] {tailargs}\n"
#: plumbum/cli/application.py:886
#, python-brace-format
msgid " {progname} [SWITCHES] {tailargs}\n"
msgstr " {progname} [OPTIES] {tailargs}\n"
#: plumbum/cli/application.py:936
msgid "; may be given multiple times"
msgstr "; kan meerdere keren gegeven worden"
#: plumbum/cli/application.py:938
msgid "; required"
msgstr "; vereist"
#: plumbum/cli/application.py:940
#, python-brace-format
msgid "; requires {0}"
msgstr "; verseist {0}"
#: plumbum/cli/application.py:947
#, python-brace-format
msgid "; excludes {0}"
msgstr "; sluit {0} uit"
#: plumbum/cli/application.py:966
#, fuzzy
msgid "Sub-commands:"
msgstr "Subcommando's"
#: plumbum/cli/application.py:1014
msgid "Prints the program's version and quits"
msgstr "Drukt de versie van het programma af en beëindigt"
#: plumbum/cli/application.py:1019
msgid "(version not set)"
msgstr "(versie niet opgegeven)"
#: plumbum/cli/switches.py:167 plumbum/cli/switches.py:225
msgid "VALUE"
msgstr "WAARDE"
#: plumbum/cli/switches.py:238
#, python-brace-format
msgid "; the default is {0}"
msgstr "; de standaard is {0}"
#: plumbum/cli/switches.py:437
#, python-brace-format
msgid "Not in range [{0:d}..{1:d}]"
msgstr "Niet binnen bereik [{0:d}..{1:d}]"
#: plumbum/cli/switches.py:546
#, python-brace-format
msgid "{0} is not a directory"
msgstr "{0} is geen map"
#: plumbum/cli/switches.py:565
#, python-brace-format
msgid "{0} is not a file"
msgstr "{0} is geen bestand"
#: plumbum/cli/switches.py:574
#, python-brace-format
msgid "{0} already exists"
msgstr "{0} bestaat al"
#, python-brace-format
#~ msgid "got unexpected keyword argument(s): {0}"
#~ msgstr "Onverwacht(e) trefwoord argument(en) gegeven: {0}"
#, python-brace-format
#~ msgid "Expected one of {0}"
#~ msgstr "Verwachtte één van {0}"

View File

@@ -0,0 +1,238 @@
# Russian translations for PACKAGE package
# Английские переводы для пакета PACKAGE.
# Copyright (C) 2017 THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# <cpp.create@gmail.com>, 2017.
#
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2022-10-05 22:39-0400\n"
"PO-Revision-Date: 2017-08-14 10:21+0200\n"
"Last-Translator: <cpp.create@gmail.com>\n"
"Language-Team: Russian <ru@li.org>\n"
"Language: ru\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n"
"%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);\n"
#: plumbum/cli/application.py:69
#, python-brace-format
msgid "Subcommand({self.name}, {self.subapplication})"
msgstr "Подкоманда({self.name}, {self.subapplication})"
#: plumbum/cli/application.py:73
msgid "Switches"
msgstr "Опции"
#: plumbum/cli/application.py:73
msgid "Meta-switches"
msgstr "Мета-опции"
#: plumbum/cli/application.py:163
#, python-brace-format
msgid "see '{parent} {sub} --help' for more info"
msgstr "вызовите '{parent} {sub} --help' для более полной справки"
#: plumbum/cli/application.py:220
#, fuzzy
msgid "Sub-command names cannot start with '-'"
msgstr "Имена подкомманд не могут начинаться с '-'"
#: plumbum/cli/application.py:238
#, python-brace-format
msgid "Switch {name} already defined and is not overridable"
msgstr "Опция {name} уже определена и не может быть переопределена"
#: plumbum/cli/application.py:343
#, python-brace-format
msgid "Ambiguous partial switch {0}"
msgstr ""
#: plumbum/cli/application.py:348 plumbum/cli/application.py:373
#: plumbum/cli/application.py:389
#, python-brace-format
msgid "Unknown switch {0}"
msgstr "Неизестная опция {0}"
#: plumbum/cli/application.py:353 plumbum/cli/application.py:362
#: plumbum/cli/application.py:381
#, python-brace-format
msgid "Switch {0} requires an argument"
msgstr "Для опции {0} необходим аргумент"
#: plumbum/cli/application.py:401
#, python-brace-format
msgid "Switch {0} already given"
msgstr "Опция {0} уже была передана"
#: plumbum/cli/application.py:403
#, python-brace-format
msgid "Switch {0} already given ({1} is equivalent)"
msgstr "Опция {0} уже была передана (эквивалентна {1})"
#: plumbum/cli/application.py:451
msgid ""
"Argument of {name} expected to be {argtype}, not {val!r}:\n"
" {ex!r}"
msgstr ""
"Аргумент опции {name} должен быть типа {argtype}, но не {val!r}:\n"
" {ex!r}"
#: plumbum/cli/application.py:470
#, python-brace-format
msgid "Switch {0} is mandatory"
msgstr "Опция {0} обязательна"
#: plumbum/cli/application.py:490
#, python-brace-format
msgid "Given {0}, the following are missing {1}"
msgstr "При передаче {0}, необходимо также указать {1}"
#: plumbum/cli/application.py:498
#, python-brace-format
msgid "Given {0}, the following are invalid {1}"
msgstr "При передаче {0}, нельзя указать {1}"
#: plumbum/cli/application.py:515
#, python-brace-format
msgid "Expected at least {0} positional argument, got {1}"
msgid_plural "Expected at least {0} positional arguments, got {1}"
msgstr[0] "Ожидая как минимум {0} позиционный аргумент, получено {1}"
msgstr[1] "Ожидая как минимум {0} позиционных аргумента, получено {1}"
msgstr[2] "Ожидая как минимум {0} позиционных аргументов, получено {1}"
#: plumbum/cli/application.py:523
#, python-brace-format
msgid "Expected at most {0} positional argument, got {1}"
msgid_plural "Expected at most {0} positional arguments, got {1}"
msgstr[0] "Ожидая как максимум {0} позиционный аргумент, получено {1}"
msgstr[1] "Ожидая как максимум {0} позиционных аргумента, получено {1}"
msgstr[2] "Ожидая как максимум {0} позиционных аргументов, получено {1}"
#: plumbum/cli/application.py:624
#, python-brace-format
msgid "Error: {0}"
msgstr "Ошибка: {0}"
#: plumbum/cli/application.py:625 plumbum/cli/application.py:711
#: plumbum/cli/application.py:716
msgid "------"
msgstr "-------"
#: plumbum/cli/application.py:694
#, python-brace-format
msgid "Switch {0} must be a sequence (iterable)"
msgstr "Опция {0} должна быть последовательностью (перечислением)"
#: plumbum/cli/application.py:699
#, python-brace-format
msgid "Switch {0} is a boolean flag"
msgstr "Опция {0} - это булев флаг"
#: plumbum/cli/application.py:710
#, python-brace-format
msgid "Unknown sub-command '{0}'"
msgstr "Неизестная подкоманда '{0}'"
#: plumbum/cli/application.py:715
msgid "No sub-command given"
msgstr "Подкоманда не задана"
#: plumbum/cli/application.py:721
msgid "main() not implemented"
msgstr "Функция main() не реализована"
#: plumbum/cli/application.py:734
#, fuzzy
msgid "Prints help messages of all sub-commands and quits"
msgstr "Печатает помощь по всем подкомандам и выходит"
#: plumbum/cli/application.py:754
msgid "Prints this help message and quits"
msgstr "Печатает это сообщение и выходит"
#: plumbum/cli/application.py:877
msgid "Usage:"
msgstr "Использование:"
#: plumbum/cli/application.py:883
#, python-brace-format
msgid " {progname} [SWITCHES] [SUBCOMMAND [SWITCHES]] {tailargs}\n"
msgstr " {progname} [ОПЦИИ] [ПОДКОМАНДА [ОПЦИИ]] {tailargs}\n"
#: plumbum/cli/application.py:886
#, python-brace-format
msgid " {progname} [SWITCHES] {tailargs}\n"
msgstr " {progname} [ОПЦИИ] {tailargs}\n"
#: plumbum/cli/application.py:936
msgid "; may be given multiple times"
msgstr "; может быть передана несколько раз"
#: plumbum/cli/application.py:938
msgid "; required"
msgstr "; обязательная"
#: plumbum/cli/application.py:940
#, python-brace-format
msgid "; requires {0}"
msgstr "; запрашивает {0}"
#: plumbum/cli/application.py:947
#, python-brace-format
msgid "; excludes {0}"
msgstr "; исключает {0}"
#: plumbum/cli/application.py:966
#, fuzzy
msgid "Sub-commands:"
msgstr "Подкоманды:"
#: plumbum/cli/application.py:1014
msgid "Prints the program's version and quits"
msgstr "Печатает версию этой программы и выходит"
#: plumbum/cli/application.py:1019
msgid "(version not set)"
msgstr "(версия не задана)"
#: plumbum/cli/switches.py:167 plumbum/cli/switches.py:225
msgid "VALUE"
msgstr "ЗНАЧЕНИЕ"
#: plumbum/cli/switches.py:238
#, python-brace-format
msgid "; the default is {0}"
msgstr "; по умолчанию - {0}"
#: plumbum/cli/switches.py:437
#, python-brace-format
msgid "Not in range [{0:d}..{1:d}]"
msgstr "Не в промежутке [{0:d}..{1:d}]"
#: plumbum/cli/switches.py:546
#, python-brace-format
msgid "{0} is not a directory"
msgstr "{0} - это не папка"
#: plumbum/cli/switches.py:565
#, python-brace-format
msgid "{0} is not a file"
msgstr "{0} - это не файл"
#: plumbum/cli/switches.py:574
#, python-brace-format
msgid "{0} already exists"
msgstr "{0} уже была передана"
#, python-brace-format
#~ msgid "got unexpected keyword argument(s): {0}"
#~ msgstr "получен(ы) неожиданный(е) аргумент(ы) ключ-значение: {0}"
#, python-brace-format
#~ msgid "Expected one of {0}"
#~ msgstr "Ожидался один из {0}"

View File

@@ -0,0 +1,112 @@
import sys
from plumbum import colors
from .. import cli
from .termsize import get_terminal_size
class Image:
__slots__ = "size char_ratio".split()
def __init__(self, size=None, char_ratio=2.45):
self.size = size
self.char_ratio = char_ratio
def best_aspect(self, orig, term):
"""Select a best possible size matching the original aspect ratio.
Size is width, height.
The char_ratio option gives the height of each char with respect
to its width, zero for no effect."""
if not self.char_ratio: # Don't use if char ratio is 0
return term
orig_ratio = orig[0] / orig[1] / self.char_ratio
if int(term[1] / orig_ratio) <= term[0]:
new_size = int(term[1] / orig_ratio), term[1]
else:
new_size = term[0], int(term[0] * orig_ratio)
return new_size
def show(self, filename, double=False):
"""Display an image on the command line. Can select a size or show in double resolution."""
import PIL.Image
return (
self.show_pil_double(PIL.Image.open(filename))
if double
else self.show_pil(PIL.Image.open(filename))
)
def _init_size(self, im):
"""Return the expected image size"""
if self.size is None:
term_size = get_terminal_size()
return self.best_aspect(im.size, term_size)
return self.size
def show_pil(self, im):
"Standard show routine"
size = self._init_size(im)
new_im = im.resize(size).convert("RGB")
for y in range(size[1]):
for x in range(size[0] - 1):
pix = new_im.getpixel((x, y))
sys.stdout.write(colors.bg.rgb(*pix) + " ") # '\u2588'
sys.stdout.write(colors.reset + " \n")
sys.stdout.write(colors.reset + "\n")
sys.stdout.flush()
def show_pil_double(self, im):
"Show double resolution on some fonts"
size = self._init_size(im)
size = (size[0], size[1] * 2)
new_im = im.resize(size).convert("RGB")
for y in range(size[1] // 2):
for x in range(size[0] - 1):
pix = new_im.getpixel((x, y * 2))
pixl = new_im.getpixel((x, y * 2 + 1))
sys.stdout.write(
(colors.bg.rgb(*pixl) & colors.fg.rgb(*pix)) + "\u2580"
)
sys.stdout.write(colors.reset + " \n")
sys.stdout.write(colors.reset + "\n")
sys.stdout.flush()
class ShowImageApp(cli.Application):
"Display an image on the terminal"
double = cli.Flag(
["-d", "--double"], help="Double resolution (looks good only with some fonts)"
)
@cli.switch(["-c", "--colors"], cli.Range(1, 4), help="Level of color, 1-4")
def colors_set(self, n): # pylint: disable=no-self-use
colors.use_color = n
size = cli.SwitchAttr(["-s", "--size"], help="Size, should be in the form 100x150")
ratio = cli.SwitchAttr(
["--ratio"], float, default=2.45, help="Aspect ratio of the font"
)
@cli.positional(cli.ExistingFile)
def main(self, filename):
size = None
if self.size:
size = map(int, self.size.split("x"))
Image(size, self.ratio).show(filename, self.double)
if __name__ == "__main__":
ShowImageApp.run()

View File

@@ -0,0 +1,272 @@
"""
Progress bar
------------
"""
import datetime
import sys
import warnings
from abc import ABC, abstractmethod
from plumbum.cli.termsize import get_terminal_size
class ProgressBase(ABC):
"""Base class for progress bars. Customize for types of progress bars.
:param iterator: The iterator to wrap with a progress bar
:param length: The length of the iterator (will use ``__len__`` if None)
:param timer: Try to time the completion status of the iterator
:param body: True if the slow portion occurs outside the iterator (in a loop, for example)
:param has_output: True if the iteration body produces output to the screen (forces rewrite off)
:param clear: Clear the progress bar afterwards, if applicable.
"""
def __init__(
self,
iterator=None,
length=None,
timer=True,
body=False,
has_output=False,
clear=True,
):
if length is None:
length = len(iterator)
elif iterator is None:
iterator = range(length)
elif length is None and iterator is None:
raise TypeError("Expected either an iterator or a length")
self.length = length
self.iterator = iterator
self.timer = timer
self.body = body
self.has_output = has_output
self.clear = clear
def __len__(self):
return self.length
def __iter__(self):
self.start()
return self
@abstractmethod
def start(self):
"""This should initialize the progress bar and the iterator"""
self.iter = iter(self.iterator)
self.value = -1 if self.body else 0
self._start_time = datetime.datetime.now()
def __next__(self):
try:
rval = next(self.iter)
self.increment()
except StopIteration:
self.done()
raise
return rval
def next(self):
return next(self)
@property
def value(self):
"""This is the current value, as a property so setting it can be customized"""
return self._value
@value.setter
def value(self, val):
self._value = val
@abstractmethod
def display(self):
"""Called to update the progress bar"""
def increment(self):
"""Sets next value and displays the bar"""
self.value += 1
self.display()
def time_remaining(self):
"""Get the time remaining for the progress bar, guesses"""
if self.value < 1:
return None, None
elapsed_time = datetime.datetime.now() - self._start_time
time_each = (
elapsed_time.days * 24 * 60 * 60
+ elapsed_time.seconds
+ elapsed_time.microseconds / 1000000.0
) / self.value
time_remaining = time_each * (self.length - self.value)
return elapsed_time, datetime.timedelta(0, time_remaining, 0)
def str_time_remaining(self):
"""Returns a string version of time remaining"""
if self.value < 1:
return "Starting... "
elapsed_time, time_remaining = list(map(str, self.time_remaining()))
completed = elapsed_time.split(".")[0]
remaining = time_remaining.split(".")[0]
return f"{completed} completed, {remaining} remaining"
@abstractmethod
def done(self):
"""Is called when the iterator is done."""
@classmethod
def range(cls, *value, **kargs):
"""Fast shortcut to create a range based progress bar, assumes work done in body"""
return cls(range(*value), body=True, **kargs)
@classmethod
def wrap(cls, iterator, length=None, **kargs):
"""Shortcut to wrap an iterator that does not do all the work internally"""
return cls(iterator, length, body=True, **kargs)
class Progress(ProgressBase):
def start(self):
super().start()
self.display()
def done(self):
self.value = self.length
self.display()
if self.clear and not self.has_output:
sys.stdout.write("\r" + len(str(self)) * " " + "\r")
else:
sys.stdout.write("\n")
sys.stdout.flush()
def __str__(self):
width = get_terminal_size(default=(0, 0))[0]
if self.length == 0:
self.width = 0
return "0/0 complete"
percent = max(self.value, 0) / self.length
ending = " " + (
self.str_time_remaining()
if self.timer
else f"{self.value} of {self.length} complete"
)
if width - len(ending) < 10 or self.has_output:
self.width = 0
if self.timer:
return f"{percent:.0%} complete: {self.str_time_remaining()}"
return f"{percent:.0%} complete"
self.width = width - len(ending) - 2 - 1
nstars = int(percent * self.width)
pbar = "[" + "*" * nstars + " " * (self.width - nstars) + "]" + ending
str_percent = f" {percent:.0%} "
return (
pbar[: self.width // 2 - 2]
+ str_percent
+ pbar[self.width // 2 + len(str_percent) - 2 :]
)
def display(self):
disptxt = str(self)
if self.width == 0 or self.has_output:
sys.stdout.write(disptxt + "\n")
else:
sys.stdout.write("\r")
sys.stdout.write(disptxt)
sys.stdout.flush()
class ProgressIPy(ProgressBase): # pragma: no cover
HTMLBOX = '<div class="widget-hbox widget-progress"><div class="widget-label" style="display:block;">{0}</div></div>'
def __init__(self, *args, **kargs):
# Ipython gives warnings when using widgets about the API potentially changing
with warnings.catch_warnings():
warnings.simplefilter("ignore")
try:
from ipywidgets import HTML, HBox, IntProgress
except ImportError: # Support IPython < 4.0
from IPython.html.widgets import HTML, HBox, IntProgress
super().__init__(*args, **kargs)
self.prog = IntProgress(max=self.length)
self._label = HTML()
self._box = HBox((self.prog, self._label))
def start(self):
from IPython.display import display
display(self._box)
super().start()
@property
def value(self):
"""This is the current value, -1 allowed (automatically fixed for display)"""
return self._value
@value.setter
def value(self, val):
self._value = val
self.prog.value = max(val, 0)
self.prog.description = f"{self.value / self.length:.2%}"
if self.timer and val > 0:
self._label.value = self.HTMLBOX.format(self.str_time_remaining())
def display(self):
pass
def done(self):
if self.clear:
self._box.close()
class ProgressAuto(ProgressBase):
"""Automatically selects the best progress bar (IPython HTML or text). Does not work with qtconsole
(as that is correctly identified as identical to notebook, since the kernel is the same); it will still
iterate, but no graphical indication will be displayed.
:param iterator: The iterator to wrap with a progress bar
:param length: The length of the iterator (will use ``__len__`` if None)
:param timer: Try to time the completion status of the iterator
:param body: True if the slow portion occurs outside the iterator (in a loop, for example)
"""
def __new__(cls, *args, **kargs):
"""Uses the generator trick that if a cls instance is returned, the __init__ method is not called."""
try: # pragma: no cover
__IPYTHON__ # pylint: disable=pointless-statement
try:
from traitlets import TraitError
except ImportError: # Support for IPython < 4.0
from IPython.utils.traitlets import TraitError
try:
return ProgressIPy(*args, **kargs)
except TraitError:
raise NameError() from None
except (NameError, ImportError):
return Progress(*args, **kargs)
ProgressAuto.register(ProgressIPy)
ProgressAuto.register(Progress)
def main():
import time
tst = Progress.range(20)
for _ in tst:
time.sleep(1)
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,575 @@
import collections.abc
import contextlib
import inspect
from abc import ABC, abstractmethod
from typing import Callable, Generator, List, Union
from plumbum import local
from plumbum.cli.i18n import get_translation_for
from plumbum.lib import getdoc
_translation = get_translation_for(__name__)
_, ngettext = _translation.gettext, _translation.ngettext
class SwitchError(Exception):
"""A general switch related-error (base class of all other switch errors)"""
class PositionalArgumentsError(SwitchError):
"""Raised when an invalid number of positional arguments has been given"""
class SwitchCombinationError(SwitchError):
"""Raised when an invalid combination of switches has been given"""
class UnknownSwitch(SwitchError):
"""Raised when an unrecognized switch has been given"""
class MissingArgument(SwitchError):
"""Raised when a switch requires an argument, but one was not provided"""
class MissingMandatorySwitch(SwitchError):
"""Raised when a mandatory switch has not been given"""
class WrongArgumentType(SwitchError):
"""Raised when a switch expected an argument of some type, but an argument of a wrong
type has been given"""
class SubcommandError(SwitchError):
"""Raised when there's something wrong with sub-commands"""
# ===================================================================================================
# The switch decorator
# ===================================================================================================
class SwitchInfo:
def __init__(self, **kwargs):
for k, v in kwargs.items():
setattr(self, k, v)
def switch(
names,
argtype=None,
argname=None,
list=False, # pylint: disable=redefined-builtin
mandatory=False,
requires=(),
excludes=(),
help=None, # pylint: disable=redefined-builtin
overridable=False,
group="Switches",
envname=None,
):
"""
A decorator that exposes functions as command-line switches. Usage::
class MyApp(Application):
@switch(["-l", "--log-to-file"], argtype = str)
def log_to_file(self, filename):
handler = logging.FileHandler(filename)
logger.addHandler(handler)
@switch(["--verbose"], excludes=["--terse"], requires=["--log-to-file"])
def set_debug(self):
logger.setLevel(logging.DEBUG)
@switch(["--terse"], excludes=["--verbose"], requires=["--log-to-file"])
def set_terse(self):
logger.setLevel(logging.WARNING)
:param names: The name(s) under which the function is reachable; it can be a string
or a list of string, but at least one name is required. There's no need
to prefix the name with ``-`` or ``--`` (this is added automatically),
but it can be used for clarity. Single-letter names are prefixed by ``-``,
while longer names are prefixed by ``--``
:param envname: Name of environment variable to extract value from, as alternative to argv
:param argtype: If this function takes an argument, you need to specify its type. The
default is ``None``, which means the function takes no argument. The type
is more of a "validator" than a real type; it can be any callable object
that raises a ``TypeError`` if the argument is invalid, or returns an
appropriate value on success. If the user provides an invalid value,
:func:`plumbum.cli.WrongArgumentType`
:param argname: The name of the argument; if ``None``, the name will be inferred from the
function's signature
:param list: Whether or not this switch can be repeated (e.g. ``gcc -I/lib -I/usr/lib``).
If ``False``, only a single occurrence of the switch is allowed; if ``True``,
it may be repeated indefinitely. The occurrences are collected into a list,
so the function is only called once with the collections. For instance,
for ``gcc -I/lib -I/usr/lib``, the function will be called with
``["/lib", "/usr/lib"]``.
:param mandatory: Whether or not this switch is mandatory; if a mandatory switch is not
given, :class:`MissingMandatorySwitch <plumbum.cli.MissingMandatorySwitch>`
is raised. The default is ``False``.
:param requires: A list of switches that this switch depends on ("requires"). This means that
it's invalid to invoke this switch without also invoking the required ones.
In the example above, it's illegal to pass ``--verbose`` or ``--terse``
without also passing ``--log-to-file``. By default, this list is empty,
which means the switch has no prerequisites. If an invalid combination
is given, :class:`SwitchCombinationError <plumbum.cli.SwitchCombinationError>`
is raised.
Note that this list is made of the switch *names*; if a switch has more
than a single name, any of its names will do.
.. note::
There is no guarantee on the (topological) order in which the actual
switch functions will be invoked, as the dependency graph might contain
cycles.
:param excludes: A list of switches that this switch forbids ("excludes"). This means that
it's invalid to invoke this switch if any of the excluded ones are given.
In the example above, it's illegal to pass ``--verbose`` along with
``--terse``, as it will result in a contradiction. By default, this list
is empty, which means the switch has no prerequisites. If an invalid
combination is given, :class:`SwitchCombinationError
<plumbum.cli.SwitchCombinationError>` is raised.
Note that this list is made of the switch *names*; if a switch has more
than a single name, any of its names will do.
:param help: The help message (description) for this switch; this description is used when
``--help`` is given. If ``None``, the function's docstring will be used.
:param overridable: Whether or not the names of this switch are overridable by other switches.
If ``False`` (the default), having another switch function with the same
name(s) will cause an exception. If ``True``, this is silently ignored.
:param group: The switch's *group*; this is a string that is used to group related switches
together when ``--help`` is given. The default group is ``Switches``.
:returns: The decorated function (with a ``_switch_info`` attribute)
"""
if isinstance(names, str):
names = [names]
names = [n.lstrip("-") for n in names]
requires = [n.lstrip("-") for n in requires]
excludes = [n.lstrip("-") for n in excludes]
def deco(func):
if argname is None:
argspec = inspect.getfullargspec(func).args
if len(argspec) == 2:
argname2 = argspec[1]
else:
argname2 = _("VALUE")
else:
argname2 = argname
help2 = getdoc(func) if help is None else help
if not help2:
help2 = str(func)
func._switch_info = SwitchInfo(
names=names,
envname=envname,
argtype=argtype,
list=list,
func=func,
mandatory=mandatory,
overridable=overridable,
group=group,
requires=requires,
excludes=excludes,
argname=argname2,
help=help2,
)
return func
return deco
def autoswitch(*args, **kwargs):
"""A decorator that exposes a function as a switch, "inferring" the name of the switch
from the function's name (converting to lower-case, and replacing underscores with hyphens).
The arguments are the same as for :func:`switch <plumbum.cli.switch>`."""
def deco(func):
return switch(func.__name__.replace("_", "-"), *args, **kwargs)(func)
return deco
# ===================================================================================================
# Switch Attributes
# ===================================================================================================
class SwitchAttr:
"""
A switch that stores its result in an attribute (descriptor). Usage::
class MyApp(Application):
logfile = SwitchAttr(["-f", "--log-file"], str)
def main(self):
if self.logfile:
open(self.logfile, "w")
:param names: The switch names
:param argtype: The switch argument's (and attribute's) type
:param default: The attribute's default value (``None``)
:param argname: The switch argument's name (default is ``"VALUE"``)
:param kwargs: Any of the keyword arguments accepted by :func:`switch <plumbum.cli.switch>`
"""
ATTR_NAME = "__plumbum_switchattr_dict__"
VALUE = _("VALUE")
def __init__(
self,
names,
argtype=str,
default=None,
list=False, # pylint: disable=redefined-builtin
argname=VALUE,
**kwargs,
):
self.__doc__ = "Sets an attribute" # to prevent the help message from showing SwitchAttr's docstring
if default and argtype is not None:
defaultmsg = _("; the default is {0}").format(default)
if "help" in kwargs:
kwargs["help"] += defaultmsg
else:
kwargs["help"] = defaultmsg.lstrip("; ")
switch(names, argtype=argtype, argname=argname, list=list, **kwargs)(self)
listtype = type([])
if list:
if default is None:
self._default_value = []
elif isinstance(default, (tuple, listtype)):
self._default_value = listtype(default)
else:
self._default_value = [default]
else:
self._default_value = default
def __call__(self, inst, val):
self.__set__(inst, val)
def __get__(self, inst, cls):
if inst is None:
return self
return getattr(inst, self.ATTR_NAME, {}).get(self, self._default_value)
def __set__(self, inst, val):
if inst is None:
raise AttributeError("cannot set an unbound SwitchAttr")
if not hasattr(inst, self.ATTR_NAME):
setattr(inst, self.ATTR_NAME, {self: val})
else:
getattr(inst, self.ATTR_NAME)[self] = val
class Flag(SwitchAttr):
"""A specialized :class:`SwitchAttr <plumbum.cli.SwitchAttr>` for boolean flags. If the flag is not
given, the value of this attribute is ``default``; if it is given, the value changes
to ``not default``. Usage::
class MyApp(Application):
verbose = Flag(["-v", "--verbose"], help = "If given, I'll be very talkative")
:param names: The switch names
:param default: The attribute's initial value (``False`` by default)
:param kwargs: Any of the keyword arguments accepted by :func:`switch <plumbum.cli.switch>`,
except for ``list`` and ``argtype``.
"""
def __init__(self, names, default=False, **kwargs):
SwitchAttr.__init__(
self, names, argtype=None, default=default, list=False, **kwargs
)
def __call__(self, inst):
self.__set__(inst, not self._default_value)
class CountOf(SwitchAttr):
"""A specialized :class:`SwitchAttr <plumbum.cli.SwitchAttr>` that counts the number of
occurrences of the switch in the command line. Usage::
class MyApp(Application):
verbosity = CountOf(["-v", "--verbose"], help = "The more, the merrier")
If ``-v -v -vv`` is given in the command-line, it will result in ``verbosity = 4``.
:param names: The switch names
:param default: The default value (0)
:param kwargs: Any of the keyword arguments accepted by :func:`switch <plumbum.cli.switch>`,
except for ``list`` and ``argtype``.
"""
def __init__(self, names, default=0, **kwargs):
SwitchAttr.__init__(
self, names, argtype=None, default=default, list=True, **kwargs
)
self._default_value = default # issue #118
def __call__(self, inst, v):
self.__set__(inst, len(v))
# ===================================================================================================
# Decorator for function that adds argument checking
# ===================================================================================================
class positional:
"""
Runs a validator on the main function for a class.
This should be used like this::
class MyApp(cli.Application):
@cli.positional(cli.Range(1,10), cli.ExistingFile)
def main(self, x, *f):
# x is a range, f's are all ExistingFile's)
Or, Python 3 only::
class MyApp(cli.Application):
def main(self, x : cli.Range(1,10), *f : cli.ExistingFile):
# x is a range, f's are all ExistingFile's)
If you do not want to validate on the annotations, use this decorator (
even if empty) to override annotation validation.
Validators should be callable, and should have a ``.choices()`` function with
possible choices. (For future argument completion, for example)
Default arguments do not go through the validator.
#TODO: Check with MyPy
"""
def __init__(self, *args, **kargs):
self.args = args
self.kargs = kargs
def __call__(self, function):
m = inspect.getfullargspec(function)
args_names = list(m.args[1:])
positional_list = [None] * len(args_names)
varargs = None
for i in range(min(len(positional_list), len(self.args))):
positional_list[i] = self.args[i]
if len(args_names) + 1 == len(self.args):
varargs = self.args[-1]
# All args are positional, so convert kargs to positional
for item, value in self.kargs.items():
if item == m.varargs:
varargs = value
else:
positional_list[args_names.index(item)] = value
function.positional = positional_list
function.positional_varargs = varargs
return function
class Validator(ABC):
__slots__ = ()
@abstractmethod
def __call__(self, obj):
"Must be implemented for a Validator to work"
def choices(self, partial=""): # pylint: disable=no-self-use, unused-argument
"""Should return set of valid choices, can be given optional partial info"""
return set()
def __repr__(self):
"""If not overridden, will print the slots as args"""
slots = {}
for cls in self.__mro__:
for prop in getattr(cls, "__slots__", ()):
if prop[0] != "_":
slots[prop] = getattr(self, prop)
mystrs = (f"{name} = {value}" for name, value in slots.items())
mystrs_str = ", ".join(mystrs)
return f"{self.__class__.__name__}({mystrs_str})"
# ===================================================================================================
# Switch type validators
# ===================================================================================================
class Range(Validator):
"""
A switch-type validator that checks for the inclusion of a value in a certain range.
Usage::
class MyApp(Application):
age = SwitchAttr(["--age"], Range(18, 120))
:param start: The minimal value
:param end: The maximal value
"""
__slots__ = ("start", "end")
def __init__(self, start, end):
self.start = start
self.end = end
def __repr__(self):
return f"[{self.start:d}..{self.end:d}]"
def __call__(self, obj):
obj = int(obj)
if obj < self.start or obj > self.end:
raise ValueError(
_("Not in range [{0:d}..{1:d}]").format(self.start, self.end)
)
return obj
def choices(self, partial=""):
# TODO: Add partial handling
return set(range(self.start, self.end + 1))
class Set(Validator):
"""
A switch-type validator that checks that the value is contained in a defined
set of values. Usage::
class MyApp(Application):
mode = SwitchAttr(["--mode"], Set("TCP", "UDP", case_sensitive = False))
num = SwitchAttr(["--num"], Set("MIN", "MAX", int, csv = True))
:param values: The set of values (strings), or other callable validators, or types,
or any other object that can be compared to a string.
:param case_sensitive: A keyword argument that indicates whether to use case-sensitive
comparison or not. The default is ``False``
:param csv: splits the input as a comma-separated-value before validating and returning
a list. Accepts ``True``, ``False``, or a string for the separator
:param all_markers: When a user inputs any value from this set, all values are iterated
over. Something like {"*", "all"} would be a potential setting for
this option.
"""
def __init__(
self,
*values: Union[str, Callable[[str], str]],
case_sensitive: bool = False,
csv: Union[bool, str] = False,
all_markers: "collections.abc.Set[str]" = frozenset(),
) -> None:
self.case_sensitive = case_sensitive
if isinstance(csv, bool):
self.csv = "," if csv else ""
else:
self.csv = csv
self.values = values
self.all_markers = all_markers
def __repr__(self):
items = ", ".join(v if isinstance(v, str) else v.__name__ for v in self.values)
return f"{{{items}}}"
def _call_iter(
self, value: str, check_csv: bool = True
) -> Generator[str, None, None]:
if self.csv and check_csv:
for v in value.split(self.csv):
yield from self._call_iter(v.strip(), check_csv=False)
if not self.case_sensitive:
value = value.lower()
for opt in self.values:
if isinstance(opt, str):
if not self.case_sensitive:
opt = opt.lower()
if opt == value or value in self.all_markers:
yield opt # always return original value
continue
with contextlib.suppress(ValueError):
yield opt(value)
def __call__(self, value: str, check_csv: bool = True) -> Union[str, List[str]]:
items = list(self._call_iter(value, check_csv))
if not items:
msg = f"Invalid value: {value} (Expected one of {self.values})"
raise ValueError(msg)
if self.csv and check_csv or len(items) > 1:
return items
return items[0]
def choices(self, partial=""):
choices = {opt if isinstance(opt, str) else f"({opt})" for opt in self.values}
choices |= self.all_markers
if partial:
choices = {opt for opt in choices if opt.lower().startswith(partial)}
return choices
CSV = Set(str, csv=True)
class Predicate:
"""A wrapper for a single-argument function with pretty printing"""
def __init__(self, func):
self.func = func
def __str__(self):
return self.func.__name__
def __call__(self, val):
return self.func(val)
def choices(self, partial=""): # pylint: disable=no-self-use, unused-argument
return set()
@Predicate
def ExistingDirectory(val):
"""A switch-type validator that ensures that the given argument is an existing directory"""
p = local.path(val)
if not p.is_dir():
raise ValueError(_("{0} is not a directory").format(val))
return p
@Predicate
def MakeDirectory(val):
p = local.path(val)
if p.is_file():
raise ValueError(f"{val} is a file, should be nonexistent, or a directory")
if not p.exists():
p.mkdir()
return p
@Predicate
def ExistingFile(val):
"""A switch-type validator that ensures that the given argument is an existing file"""
p = local.path(val)
if not p.is_file():
raise ValueError(_("{0} is not a file").format(val))
return p
@Predicate
def NonexistentPath(val):
"""A switch-type validator that ensures that the given argument is a nonexistent path"""
p = local.path(val)
if p.exists():
raise ValueError(_("{0} already exists").format(val))
return p

View File

@@ -0,0 +1,244 @@
"""
Terminal-related utilities
--------------------------
"""
import contextlib
import os
import sys
from typing import List, Optional
from plumbum import local
from .progress import Progress
from .termsize import get_terminal_size
__all__ = [
"readline",
"ask",
"choose",
"prompt",
"get_terminal_size",
"Progress",
"get_terminal_size",
]
def __dir__() -> List[str]:
return __all__
def readline(message: str = "") -> str:
"""Gets a line of input from the user (stdin)"""
sys.stdout.write(message)
sys.stdout.flush()
return sys.stdin.readline()
def ask(question: str, default: Optional[bool] = None) -> bool:
"""
Presents the user with a yes/no question.
:param question: The question to ask
:param default: If ``None``, the user must answer. If ``True`` or ``False``, lack of response is
interpreted as the default option
:returns: the user's choice
"""
question = question.rstrip().rstrip("?").rstrip() + "?"
if default is None:
question += " (y/n) "
elif default:
question += " [Y/n] "
else:
question += " [y/N] "
while True:
try:
answer = readline(question).strip().lower()
except EOFError:
answer = None
if answer in {"y", "yes"}:
return True
if answer in {"n", "no"}:
return False
if not answer and default is not None:
return default
sys.stdout.write("Invalid response, please try again\n")
def choose(question, options, default=None):
"""Prompts the user with a question and a set of options, from which the user needs to choose.
:param question: The question to ask
:param options: A set of options. It can be a list (of strings or two-tuples, mapping text
to returned-object) or a dict (mapping text to returned-object).``
:param default: If ``None``, the user must answer. Otherwise, lack of response is interpreted
as this answer
:returns: The user's choice
Example::
ans = choose("What is your favorite color?", ["blue", "yellow", "green"], default = "yellow")
# `ans` will be one of "blue", "yellow" or "green"
ans = choose("What is your favorite color?",
{"blue" : 0x0000ff, "yellow" : 0xffff00 , "green" : 0x00ff00}, default = 0x00ff00)
# this will display "blue", "yellow" and "green" but return a numerical value
"""
if hasattr(options, "items"):
options = options.items()
sys.stdout.write(question.rstrip() + "\n")
choices = {}
defindex = None
for i, item in enumerate(options):
i += 1
if isinstance(item, (tuple, list)) and len(item) == 2:
text = item[0]
val = item[1]
else:
text = item
val = item
choices[i] = val
if default is not None and default == val:
defindex = i
sys.stdout.write(f"({i}) {text}\n")
if default is not None:
if defindex is None:
msg = f"Choice [{default}]: "
else:
msg = f"Choice [{defindex}]: "
else:
msg = "Choice: "
while True:
try:
choice = readline(msg).strip()
except EOFError:
choice = ""
if not choice and default:
return default
try:
choice = int(choice)
if choice not in choices:
raise ValueError()
except ValueError:
sys.stdout.write("Invalid choice, please try again\n")
continue
return choices[choice]
def prompt(
question,
type=str, # pylint: disable=redefined-builtin
default=NotImplemented,
validator=lambda val: True,
):
"""
Presents the user with a validated question, keeps asking if validation does not pass.
:param question: The question to ask
:param type: The type of the answer, defaults to str
:param default: The default choice
:param validator: An extra validator called after type conversion, can raise ValueError or return False to trigger a retry.
:returns: the user's choice
"""
question = question.rstrip(" \t:")
if default is not NotImplemented:
question += f" [{default}]"
question += ": "
while True:
try:
ans = readline(question).strip()
except EOFError:
ans = ""
if not ans:
if default is not NotImplemented:
# sys.stdout.write("\b%s\n" % (default,))
return default
continue
try:
ans = type(ans)
except (TypeError, ValueError) as ex:
sys.stdout.write(f"Invalid value ({ex}), please try again\n")
continue
try:
valid = validator(ans)
except ValueError as ex:
sys.stdout.write(f"{ex}, please try again\n")
continue
if not valid:
sys.stdout.write("Value not in specified range, please try again\n")
continue
return ans
def hexdump(data_or_stream, bytes_per_line=16, aggregate=True):
"""Convert the given bytes (or a stream with a buffering ``read()`` method) to hexdump-formatted lines,
with possible aggregation of identical lines. Returns a generator of formatted lines.
"""
if hasattr(data_or_stream, "read"):
def read_chunk():
while True:
buf = data_or_stream.read(bytes_per_line)
if not buf:
break
yield buf
else:
def read_chunk():
for i in range(0, len(data_or_stream), bytes_per_line):
yield data_or_stream[i : i + bytes_per_line]
prev = None
skipped = False
for i, chunk in enumerate(read_chunk()):
hexd = " ".join(f"{ord(ch):02x}" for ch in chunk)
text = "".join(ch if 32 <= ord(ch) < 127 else "." for ch in chunk)
if aggregate and prev == chunk:
skipped = True
continue
prev = chunk
if skipped:
yield "*"
hexd_ljust = hexd.ljust(bytes_per_line * 3, " ")
yield f"{i*bytes_per_line:06x} | {hexd_ljust}| {text}"
skipped = False
def pager(rows, pagercmd=None): # pragma: no cover
"""Opens a pager (e.g., ``less``) to display the given text. Requires a terminal.
:param rows: a ``bytes`` or a list/iterator of "rows" (``bytes``)
:param pagercmd: the pager program to run. Defaults to ``less -RSin``
"""
if not pagercmd:
pagercmd = local["less"]["-RSin"]
if hasattr(rows, "splitlines"):
rows = rows.splitlines()
pg = pagercmd.popen(stdout=None, stderr=None)
try:
for row in rows:
line = f"{row}\n"
try:
pg.stdin.write(line)
pg.stdin.flush()
except OSError:
break
pg.stdin.close()
pg.wait()
finally:
with contextlib.suppress(Exception):
rows.close()
if pg and pg.poll() is None:
with contextlib.suppress(Exception):
pg.terminate()
os.system("reset")

View File

@@ -0,0 +1,100 @@
"""
Terminal size utility
---------------------
"""
import contextlib
import os
import platform
import warnings
from struct import Struct
from typing import Optional, Tuple
from plumbum import local
def get_terminal_size(default: Tuple[int, int] = (80, 25)) -> Tuple[int, int]:
"""
Get width and height of console; works on linux, os x, windows and cygwin
Adapted from https://gist.github.com/jtriley/1108174
Originally from: http://stackoverflow.com/questions/566746/how-to-get-console-window-width-in-python
"""
current_os = platform.system()
if current_os == "Windows": # pragma: no cover
size = _get_terminal_size_windows()
if not size:
# needed for window's python in cygwin's xterm!
size = _get_terminal_size_tput()
elif current_os in ("Linux", "Darwin", "FreeBSD", "SunOS") or current_os.startswith(
"CYGWIN"
):
size = _get_terminal_size_linux()
else: # pragma: no cover
warnings.warn(
"Plumbum does not know the type of the current OS for term size, defaulting to UNIX"
)
size = _get_terminal_size_linux()
if (
size is None
): # we'll assume the standard 80x25 if for any reason we don't know the terminal size
size = default
return size
def _get_terminal_size_windows(): # pragma: no cover
try:
from ctypes import create_string_buffer, windll
STDERR_HANDLE = -12
h = windll.kernel32.GetStdHandle(STDERR_HANDLE)
csbi_struct = Struct("hhhhHhhhhhh")
csbi = create_string_buffer(csbi_struct.size)
res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
if res:
_, _, _, _, _, left, top, right, bottom, _, _ = csbi_struct.unpack(csbi.raw)
return right - left + 1, bottom - top + 1
return None
except Exception:
return None
def _get_terminal_size_tput(): # pragma: no cover
# get terminal width
# src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window
try:
tput = local["tput"]
cols = int(tput("cols"))
rows = int(tput("lines"))
return (cols, rows)
except Exception:
return None
def _ioctl_GWINSZ(fd: int) -> Optional[Tuple[int, int]]:
yx = Struct("hh")
try:
import fcntl
import termios
# TODO: Clean this up. Problems could be hidden by the broad except.
return yx.unpack(fcntl.ioctl(fd, termios.TIOCGWINSZ, b"1234")) # type: ignore[return-value]
except Exception:
return None
def _get_terminal_size_linux() -> Optional[Tuple[int, int]]:
cr = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(1) or _ioctl_GWINSZ(2)
if not cr:
with contextlib.suppress(Exception):
fd = os.open(os.ctermid(), os.O_RDONLY)
cr = _ioctl_GWINSZ(fd)
os.close(fd)
if not cr:
try:
cr = (int(os.environ["LINES"]), int(os.environ["COLUMNS"]))
except Exception:
return None
return cr[1], cr[0]

View File

@@ -0,0 +1,9 @@
import plumbum
def __getattr__(name: str) -> plumbum.machines.LocalCommand:
"""The module-hack that allows us to use ``from plumbum.cmd import some_program``"""
try:
return plumbum.local[name]
except plumbum.CommandNotFound:
raise AttributeError(name) from None

View File

@@ -0,0 +1,44 @@
"""\
The ``ansicolor`` object provides ``bg`` and ``fg`` to access colors,
and attributes like bold and
underlined text. It also provides ``reset`` to recover the normal font.
"""
import sys
from .factories import StyleFactory
from .styles import ANSIStyle, ColorNotFound, HTMLStyle, Style
__all__ = (
"ANSIStyle",
"ColorNotFound",
"HTMLStyle",
"Style",
"StyleFactory",
"ansicolors",
"htmlcolors",
"load_ipython_extension",
"main",
)
ansicolors = StyleFactory(ANSIStyle)
htmlcolors = StyleFactory(HTMLStyle)
def load_ipython_extension(ipython): # pragma: no cover
try:
from ._ipython_ext import OutputMagics # pylint:disable=import-outside-toplevel
except ImportError:
print("IPython required for the IPython extension to be loaded.") # noqa: T201
raise
ipython.push({"colors": htmlcolors})
ipython.register_magics(OutputMagics)
def main(): # pragma: no cover
"""Color changing script entry. Call using
python3 -m plumbum.colors, will reset if no arguments given."""
color = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else ""
ansicolors.use_color = True
ansicolors.get_colors_from_string(color).now()

View File

@@ -0,0 +1,10 @@
"""
This is provided as a quick way to recover your terminal. Simply run
``python3 -m plumbum.colorlib``
to recover terminal color.
"""
from . import main
main()

View File

@@ -0,0 +1,30 @@
import sys
from io import StringIO
import IPython.display
from IPython.core.magic import Magics, cell_magic, magics_class, needs_local_scope
valid_choices = [x[8:] for x in dir(IPython.display) if "display_" == x[:8]]
@magics_class
class OutputMagics(Magics): # pragma: no cover
@needs_local_scope
@cell_magic
def to(self, line, cell, local_ns=None):
choice = line.strip()
assert choice in valid_choices, "Valid choices for '%%to' are: " + str(
valid_choices
)
display_fn = getattr(IPython.display, "display_" + choice)
# Captures stdout and renders it in the notebook
with StringIO() as out:
old_out = sys.stdout
try:
sys.stdout = out
exec(cell, self.shell.user_ns, local_ns) # pylint: disable=exec-used
out.seek(0)
display_fn(out.getvalue(), raw=True)
finally:
sys.stdout = old_out

View File

@@ -0,0 +1,208 @@
"""
Color-related factories. They produce Styles.
"""
import functools
import operator
import sys
from typing import Any
from .names import color_names, default_styles
from .styles import ColorNotFound
__all__ = ["ColorFactory", "StyleFactory"]
class ColorFactory:
"""This creates color names given fg = True/False. It usually will
be called as part of a StyleFactory."""
def __init__(self, fg, style):
self._fg = fg
self._style = style
self.reset = style.from_color(style.color_class(fg=fg))
# Adding the color name shortcuts for foreground colors
for item in color_names[:16]:
setattr(
self, item, style.from_color(style.color_class.from_simple(item, fg=fg))
)
def __getattr__(self, item):
"""Full color names work, but do not populate __dir__."""
try:
return self._style.from_color(self._style.color_class(item, fg=self._fg))
except ColorNotFound:
raise AttributeError(item) from None
def full(self, name):
"""Gets the style for a color, using standard name procedure: either full
color name, html code, or number."""
return self._style.from_color(
self._style.color_class.from_full(name, fg=self._fg)
)
def simple(self, name):
"""Return the extended color scheme color for a value or name."""
return self._style.from_color(
self._style.color_class.from_simple(name, fg=self._fg)
)
def rgb(self, r, g=None, b=None):
"""Return the extended color scheme color for a value."""
if g is None and b is None:
return self.hex(r)
return self._style.from_color(self._style.color_class(r, g, b, fg=self._fg))
def hex(self, hexcode):
"""Return the extended color scheme color for a value."""
return self._style.from_color(
self._style.color_class.from_hex(hexcode, fg=self._fg)
)
def ansi(self, ansiseq):
"""Make a style from an ansi text sequence"""
return self._style.from_ansi(ansiseq)
def __getitem__(self, val):
"""\
Shortcut to provide way to access colors numerically or by slice.
If end <= 16, will stay to simple ANSI version."""
if isinstance(val, slice):
(start, stop, stride) = val.indices(256)
if stop <= 16:
return [self.simple(v) for v in range(start, stop, stride)]
return [self.full(v) for v in range(start, stop, stride)]
if isinstance(val, tuple):
return self.rgb(*val)
try:
return self.full(val)
except ColorNotFound:
return self.hex(val)
def __call__(self, val_or_r=None, g=None, b=None):
"""Shortcut to provide way to access colors."""
if val_or_r is None or (isinstance(val_or_r, str) and val_or_r == ""):
return self._style()
if isinstance(val_or_r, self._style):
return self._style(val_or_r)
if isinstance(val_or_r, str) and "\033" in val_or_r:
return self.ansi(val_or_r)
return self._style.from_color(
self._style.color_class(val_or_r, g, b, fg=self._fg)
)
def __iter__(self):
"""Iterates through all colors in extended colorset."""
return (self.full(i) for i in range(256))
def __invert__(self):
"""Allows clearing a color with ~"""
return self.reset
def __enter__(self):
"""This will reset the color on leaving the with statement."""
return self
def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None:
"""This resets a FG/BG color or all styles,
due to different definition of RESET for the
factories."""
self.reset.now()
def __repr__(self):
"""Simple representation of the class by name."""
return f"<{self.__class__.__name__}>"
class StyleFactory(ColorFactory):
"""Factory for styles. Holds font styles, FG and BG objects representing colors, and
imitates the FG ColorFactory to a large degree."""
def __init__(self, style):
super().__init__(True, style)
self.fg = ColorFactory(True, style)
self.bg = ColorFactory(False, style)
self.do_nothing = style()
self.reset = style(reset=True)
for item in style.attribute_names:
setattr(self, item, style(attributes={item: True}))
self.load_stylesheet(default_styles)
@property
def use_color(self):
"""Shortcut for setting color usage on Style"""
return self._style.use_color
@use_color.setter
def use_color(self, val):
self._style.use_color = val
def from_ansi(self, ansi_sequence):
"""Calling this is a shortcut for creating a style from an ANSI sequence."""
return self._style.from_ansi(ansi_sequence)
@property
def stdout(self):
"""This is a shortcut for getting stdout from a class without an instance."""
return self._style._stdout if self._style._stdout is not None else sys.stdout
@stdout.setter
def stdout(self, newout):
self._style._stdout = newout
def get_colors_from_string(self, color=""):
"""
Sets color based on string, use `.` or space for separator,
and numbers, fg/bg, htmlcodes, etc all accepted (as strings).
"""
names = color.replace(".", " ").split()
prev = self
styleslist = []
for name in names:
try:
prev = getattr(prev, name)
except AttributeError:
try:
prev = prev(int(name))
except (ColorNotFound, ValueError):
prev = prev(name)
if isinstance(prev, self._style):
styleslist.append(prev)
prev = self
if styleslist:
prev = functools.reduce(operator.and_, styleslist)
return prev if isinstance(prev, self._style) else prev.reset
def filter(self, colored_string):
"""Filters out colors in a string, returning only the name."""
if isinstance(colored_string, self._style):
return colored_string
return self._style.string_filter_ansi(colored_string)
def contains_colors(self, colored_string):
"""Checks to see if a string contains colors."""
return self._style.string_contains_colors(colored_string)
def extract(self, colored_string):
"""Gets colors from an ansi string, returns those colors"""
return self._style.from_ansi(colored_string, True)
def load_stylesheet(self, stylesheet=None):
if stylesheet is None:
stylesheet = default_styles
for item in stylesheet:
setattr(self, item, self.get_colors_from_string(stylesheet[item]))

View File

@@ -0,0 +1,428 @@
"""
Names for the standard and extended color set.
Extended set is similar to `vim wiki <http://vim.wikia.com/wiki/Xterm256_color_names_for_console_Vim>`_, `colored <https://pypi.python.org/pypi/colored>`_, etc. Colors based on `wikipedia <https://en.wikipedia.org/wiki/ANSI_escape_code#Colors>`_.
You can access the index of the colors with names.index(name). You can access the
rgb values with ``r=int(html[n][1:3],16)``, etc.
"""
from typing import Tuple
color_names = """\
black
red
green
yellow
blue
magenta
cyan
light_gray
dark_gray
light_red
light_green
light_yellow
light_blue
light_magenta
light_cyan
white
grey_0
navy_blue
dark_blue
blue_3
blue_3a
blue_1
dark_green
deep_sky_blue_4
deep_sky_blue_4a
deep_sky_blue_4b
dodger_blue_3
dodger_blue_2
green_4
spring_green_4
turquoise_4
deep_sky_blue_3
deep_sky_blue_3a
dodger_blue_1
green_3
spring_green_3
dark_cyan
light_sea_green
deep_sky_blue_2
deep_sky_blue_1
green_3a
spring_green_3a
spring_green_2
cyan_3
dark_turquoise
turquoise_2
green_1
spring_green_2a
spring_green_1
medium_spring_green
cyan_2
cyan_1
dark_red
deep_pink_4
purple_4
purple_4a
purple_3
blue_violet
orange_4
grey_37
medium_purple_4
slate_blue_3
slate_blue_3a
royal_blue_1
chartreuse_4
dark_sea_green_4
pale_turquoise_4
steel_blue
steel_blue_3
cornflower_blue
chartreuse_3
dark_sea_green_4a
cadet_blue
cadet_blue_a
sky_blue_3
steel_blue_1
chartreuse_3a
pale_green_3
sea_green_3
aquamarine_3
medium_turquoise
steel_blue_1a
chartreuse_2a
sea_green_2
sea_green_1
sea_green_1a
aquamarine_1
dark_slate_gray_2
dark_red_a
deep_pink_4a
dark_magenta
dark_magenta_a
dark_violet
purple
orange_4a
light_pink_4
plum_4
medium_purple_3
medium_purple_3a
slate_blue_1
yellow_4
wheat_4
grey_53
light_slate_grey
medium_purple
light_slate_blue
yellow_4_a
dark_olive_green_3
dark_sea_green
light_sky_blue_3
light_sky_blue_3a
sky_blue_2
chartreuse_2
dark_olive_green_3a
pale_green_3a
dark_sea_green_3
dark_slate_gray_3
sky_blue_1
chartreuse_1
light_green_a
light_green_b
pale_green_1
aquamarine_1a
dark_slate_gray_1
red_3
deep_pink_4b
medium_violet_red
magenta_3
dark_violet_a
purple_a
dark_orange_3
indian_red
hot_pink_3
medium_orchid_3
medium_orchid
medium_purple_2
dark_goldenrod
light_salmon_3
rosy_brown
grey_63
medium_purple_2a
medium_purple_1
gold_3
dark_khaki
navajo_white_3
grey_69
light_steel_blue_3
light_steel_blue
yellow_3
dark_olive_green_3b
dark_sea_green_3a
dark_sea_green_2
light_cyan_3
light_sky_blue_1
green_yellow
dark_olive_green_2
pale_green_1a
dark_sea_green_2a
dark_sea_green_1
pale_turquoise_1
red_3a
deep_pink_3
deep_pink_3a
magenta_3a
magenta_3b
magenta_2
dark_orange_3a
indian_red_a
hot_pink_3a
hot_pink_2
orchid
medium_orchid_1
orange_3
light_salmon_3a
light_pink_3
pink_3
plum_3
violet
gold_3a
light_goldenrod_3
tan
misty_rose_3
thistle_3
plum_2
yellow_3a
khaki_3
light_goldenrod_2
light_yellow_3
grey_84
light_steel_blue_1
yellow_2
dark_olive_green_1
dark_olive_green_1a
dark_sea_green_1a
honeydew_2
light_cyan_1
red_1
deep_pink_2
deep_pink_1
deep_pink_1a
magenta_2a
magenta_1
orange_red_1
indian_red_1
indian_red_1a
hot_pink
hot_pink_a
medium_orchid_1a
dark_orange
salmon_1
light_coral
pale_violet_red_1
orchid_2
orchid_1
orange_1
sandy_brown
light_salmon_1
light_pink_1
pink_1
plum_1
gold_1
light_goldenrod_2a
light_goldenrod_2b
navajo_white_1
misty_rose_1
thistle_1
yellow_1
light_goldenrod_1
khaki_1
wheat_1
cornsilk_1
grey_10_0
grey_3
grey_7
grey_11
grey_15
grey_19
grey_23
grey_27
grey_30
grey_35
grey_39
grey_42
grey_46
grey_50
grey_54
grey_58
grey_62
grey_66
grey_70
grey_74
grey_78
grey_82
grey_85
grey_89
grey_93""".split()
EMPTY_SLICE = slice(None, None, None)
_greys = (
3.4,
7.4,
11,
15,
19,
23,
26.7,
30.49,
34.6,
38.6,
42.4,
46.4,
50,
54,
58,
62,
66,
69.8,
73.8,
77.7,
81.6,
85.3,
89.3,
93,
)
_grey_vals = [int(x / 100.0 * 16 * 16) for x in _greys]
_grey_html = ["#" + format(x, "02x") * 3 for x in _grey_vals]
_normals = [int(x, 16) for x in "0 5f 87 af d7 ff".split()]
_normal_html = [
"#"
+ format(_normals[n // 36], "02x")
+ format(_normals[n // 6 % 6], "02x")
+ format(_normals[n % 6], "02x")
for n in range(16 - 16, 232 - 16)
]
_base_pattern = [(n // 4, n // 2 % 2, n % 2) for n in range(8)]
_base_html = (
[f"#{x[2] * 192:02x}{x[1] * 192:02x}{x[0] * 192:02x}" for x in _base_pattern]
+ ["#808080"]
+ [f"#{x[2] * 255:02x}{x[1] * 255:02x}{x[0] * 255:02x}" for x in _base_pattern][1:]
)
color_html = _base_html + _normal_html + _grey_html
color_codes_simple = list(range(8)) + list(range(60, 68))
"""Simple colors, remember that reset is #9, second half is non as common."""
# Attributes
attributes_ansi = dict(
bold=1,
dim=2,
italics=3,
underline=4,
reverse=7,
hidden=8,
strikeout=9,
)
# Stylesheet
default_styles = dict(
warn="fg red",
title="fg cyan underline bold",
fatal="fg red bold",
highlight="bg yellow",
info="fg blue",
success="fg green",
)
# Functions to be used for color name operations
class FindNearest:
"""This is a class for finding the nearest color given rgb values.
Different find methods are available."""
def __init__(self, r: int, g: int, b: int) -> None:
self.r = r
self.b = b
self.g = g
def only_basic(self):
"""This will only return the first 8 colors!
Breaks the colorspace into cubes, returns color"""
midlevel = 0x40 # Since bright is not included
# The colors are organised so that it is a
# 3D cube, black at 0,0,0, white at 1,1,1
# Compressed to linear_integers r,g,b
# [[[0,1],[2,3]],[[4,5],[6,7]]]
# r*1 + g*2 + b*4
return (
(self.r >= midlevel) * 1
+ (self.g >= midlevel) * 2
+ (self.b >= midlevel) * 4
)
def all_slow(self, color_slice: slice = EMPTY_SLICE) -> int:
"""This is a slow way to find the nearest color."""
distances = [
self._distance_to_color(color) for color in color_html[color_slice]
]
return min(range(len(distances)), key=distances.__getitem__)
def _distance_to_color(self, color: str) -> int:
"""This computes the distance to a color, should be minimized."""
rgb = (int(color[1:3], 16), int(color[3:5], 16), int(color[5:7], 16))
return (self.r - rgb[0]) ** 2 + (self.g - rgb[1]) ** 2 + (self.b - rgb[2]) ** 2
def _distance_to_color_number(self, n: int) -> int:
color = color_html[n]
return self._distance_to_color(color)
def only_colorblock(self) -> int:
"""This finds the nearest color based on block system, only works
for 17-232 color values."""
rint = min(
range(len(_normals)), key=[abs(x - self.r) for x in _normals].__getitem__
)
bint = min(
range(len(_normals)), key=[abs(x - self.b) for x in _normals].__getitem__
)
gint = min(
range(len(_normals)), key=[abs(x - self.g) for x in _normals].__getitem__
)
return 16 + 36 * rint + 6 * gint + bint
def only_simple(self) -> int:
"""Finds the simple color-block color."""
return self.all_slow(slice(0, 16, None))
def only_grey(self) -> int:
"""Finds the greyscale color."""
rawval = (self.r + self.b + self.g) / 3
n = min(
range(len(_grey_vals)),
key=[abs(x - rawval) for x in _grey_vals].__getitem__,
)
return n + 232
def all_fast(self) -> int:
"""Runs roughly 8 times faster than the slow version."""
colors = [self.only_simple(), self.only_colorblock(), self.only_grey()]
distances = [self._distance_to_color_number(n) for n in colors]
return colors[min(range(len(distances)), key=distances.__getitem__)]
def from_html(color: str) -> Tuple[int, int, int]:
"""Convert html hex code to rgb."""
if len(color) != 7 or color[0] != "#":
raise ValueError("Invalid length of html code")
return (int(color[1:3], 16), int(color[3:5], 16), int(color[5:7], 16))
def to_html(r, g, b):
"""Convert rgb to html hex code."""
return f"#{r:02x}{g:02x}{b:02x}"

View File

@@ -0,0 +1,775 @@
"""
This file provides two classes, `Color` and `Style`.
``Color`` is rarely used directly,
but merely provides the workhorse for finding and manipulating colors.
With the ``Style`` class, any color can be directly called or given to a with statement.
"""
import contextlib
import os
import platform
import re
import sys
from abc import ABCMeta, abstractmethod
from copy import copy
from typing import IO, Dict, Optional, Union
from .names import (
FindNearest,
attributes_ansi,
color_codes_simple,
color_html,
color_names,
from_html,
)
__all__ = [
"Color",
"Style",
"ANSIStyle",
"HTMLStyle",
"ColorNotFound",
"AttributeNotFound",
]
_lower_camel_names = [n.replace("_", "") for n in color_names]
def get_color_repr():
"""Gets best colors for current system."""
if "NO_COLOR" in os.environ:
return 0
if os.environ.get("FORCE_COLOR", "") in {"0", "1", "2", "3", "4"}:
return int(os.environ["FORCE_COLOR"])
if not sys.stdout.isatty():
return 0
term = os.environ.get("TERM", "")
# Some terminals set TERM=xterm for compatibility
if term.endswith("256color") or term == "xterm":
return 3 if platform.system() == "Darwin" else 4
if term.endswith("16color"):
return 2
if term == "screen":
return 1
if os.name == "nt":
return 0
return 3
class ColorNotFound(Exception):
"""Thrown when a color is not valid for a particular method."""
class AttributeNotFound(Exception):
"""Similar to color not found, only for attributes."""
class ResetNotSupported(Exception):
"""An exception indicating that Reset is not available
for this Style."""
class Color:
"""\
Loaded with ``(r, g, b, fg)`` or ``(color, fg=fg)``. The second signature is a short cut
and will try full and hex loading.
This class stores the idea of a color, rather than a specific implementation.
It provides as many different tools for representations as possible, and can be subclassed
to add more representations, though that should not be needed for most situations. ``.from_`` class methods provide quick ways to create colors given different representations.
You will not usually interact with this class.
Possible colors::
reset = Color() # The reset color by default
background_reset = Color(fg=False) # Can be a background color
blue = Color(0,0,255) # Red, Green, Blue
green = Color.from_full("green") # Case insensitive name, from large colorset
red = Color.from_full(1) # Color number
white = Color.from_html("#FFFFFF") # HTML supported
yellow = Color.from_simple("red") # Simple colorset
The attributes are:
.. data:: reset
True it this is a reset color (following attributes don't matter if True)
.. data:: rgb
The red/green/blue tuple for this color
.. data:: simple
If true will stay to 16 color mode.
.. data:: number
The color number given the mode, closest to rgb
if not rgb not exact, gives position of closest name.
.. data:: fg
This is a foreground color if True. Background color if False.
"""
__slots__ = ("fg", "isreset", "rgb", "number", "representation", "exact")
def __init__(self, r_or_color=None, g=None, b=None, fg=True):
"""This works from color values, or tries to load non-simple ones."""
if isinstance(r_or_color, type(self)):
for item in ("fg", "isreset", "rgb", "number", "representation", "exact"):
setattr(self, item, getattr(r_or_color, item))
return
self.fg = fg
self.isreset = True # Starts as reset color
self.rgb = (0, 0, 0)
self.number = None
"Number of the original color, or closest color"
self.representation = 4
"0 for off, 1 for 8 colors, 2 for 16 colors, 3 for 256 colors, 4 for true color"
self.exact = True
"This is false if the named color does not match the real color"
if None in (g, b):
if not r_or_color:
return
try:
self._from_simple(r_or_color)
except ColorNotFound:
try:
self._from_full(r_or_color)
except ColorNotFound:
self._from_hex(r_or_color)
elif None not in (r_or_color, g, b):
self.rgb = (r_or_color, g, b)
self._init_number()
else:
raise ColorNotFound("Invalid parameters for a color!")
def _init_number(self):
"""Should always be called after filling in r, g, b, and representation.
Color will not be a reset color anymore."""
if self.representation in (0, 1):
number = FindNearest(*self.rgb).only_basic()
elif self.representation == 2:
number = FindNearest(*self.rgb).only_simple()
elif self.representation in (3, 4):
number = FindNearest(*self.rgb).all_fast()
if self.number is None:
self.number = number
self.isreset = False
self.exact = self.rgb == from_html(color_html[self.number])
if not self.exact:
self.number = number
@classmethod
def from_simple(cls, color, fg=True):
"""Creates a color from simple name or color number"""
self = cls(fg=fg)
self._from_simple(color)
return self
def _from_simple(self, color):
with contextlib.suppress(AttributeError):
color = color.lower()
color = color.replace(" ", "")
color = color.replace("_", "")
if color == "reset":
return
if color in _lower_camel_names[:16]:
self.number = _lower_camel_names.index(color)
self.rgb = from_html(color_html[self.number])
elif isinstance(color, int) and 0 <= color < 16:
self.number = color
self.rgb = from_html(color_html[color])
else:
raise ColorNotFound("Did not find color: " + repr(color))
self.representation = 2
self._init_number()
@classmethod
def from_full(cls, color, fg=True):
"""Creates a color from full name or color number"""
self = cls(fg=fg)
self._from_full(color)
return self
def _from_full(self, color):
with contextlib.suppress(AttributeError):
color = color.lower()
color = color.replace(" ", "")
color = color.replace("_", "")
if color == "reset":
return
if color in _lower_camel_names:
self.number = _lower_camel_names.index(color)
self.rgb = from_html(color_html[self.number])
elif isinstance(color, int) and 0 <= color <= 255:
self.number = color
self.rgb = from_html(color_html[color])
else:
raise ColorNotFound("Did not find color: " + repr(color))
self.representation = 3
self._init_number()
@classmethod
def from_hex(cls, color, fg=True):
"""Converts #123456 values to colors."""
self = cls(fg=fg)
self._from_hex(color)
return self
def _from_hex(self, color):
try:
self.rgb = from_html(color)
except (TypeError, ValueError):
raise ColorNotFound("Did not find htmlcode: " + repr(color)) from None
self.representation = 4
self._init_number()
@property
def name(self):
"""The (closest) name of the current color"""
return "reset" if self.isreset else color_names[self.number]
@property
def name_camelcase(self):
"""The camelcase name of the color"""
return self.name.replace("_", " ").title().replace(" ", "")
def __repr__(self):
"""This class has a smart representation that shows name and color (if not unique)."""
name = ["Deactivated:", " Basic:", "", " Full:", " True:"][self.representation]
name += "" if self.fg else " Background"
name += " " + self.name_camelcase
name += "" if self.exact else " " + self.hex_code
return name[1:]
def __eq__(self, other):
"""Reset colors are equal, otherwise rgb have to match."""
return other.isreset if self.isreset else self.rgb == other.rgb
@property
def ansi_sequence(self):
"""This is the ansi sequence as a string, ready to use."""
return "\033[" + ";".join(map(str, self.ansi_codes)) + "m"
@property
def ansi_codes(self):
"""This is the full ANSI code, can be reset, simple, 256, or full color."""
ansi_addition = 30 if self.fg else 40
if self.isreset:
return (ansi_addition + 9,)
if self.representation < 3:
return (color_codes_simple[self.number] + ansi_addition,)
if self.representation == 3:
return (ansi_addition + 8, 5, self.number)
return (ansi_addition + 8, 2, self.rgb[0], self.rgb[1], self.rgb[2])
@property
def hex_code(self):
"""This is the hex code of the current color, html style notation."""
return (
"#000000"
if self.isreset
else f"#{self.rgb[0]:02X}{self.rgb[1]:02X}{self.rgb[2]:02X}"
)
def __str__(self):
"""This just prints it's simple name"""
return self.name
def to_representation(self, val):
"""Converts a color to any representation"""
other = copy(self)
other.representation = val
if self.isreset:
return other
other.number = None
other._init_number()
return other
def limit_representation(self, val):
"""Only converts if val is lower than representation"""
return self if self.representation <= val else self.to_representation(val)
class Style(metaclass=ABCMeta):
"""This class allows the color changes to be called directly
to write them to stdout, ``[]`` calls to wrap colors (or the ``.wrap`` method)
and can be called in a with statement.
"""
__slots__ = ("attributes", "fg", "bg", "isreset", "__weakref__")
color_class = Color
"""The class of color to use. Never hardcode ``Color`` call when writing a Style
method."""
attribute_names: Union[Dict[str, str], Dict[str, int]]
_stdout: Optional[IO] = None
end = "\n"
"""The endline character. Override if needed in subclasses."""
ANSI_REG = re.compile("\033" + r"\[([\d;]+)m")
"""The regular expression that finds ansi codes in a string."""
@property
def stdout(self):
"""\
This property will allow custom, class level control of stdout.
It will use current sys.stdout if set to None (default).
Unfortunately, it only works on an instance..
"""
# Import sys repeated here to make calling this stable in atexit function
import sys # pylint: disable=reimported, redefined-outer-name, import-outside-toplevel
return (
self.__class__._stdout if self.__class__._stdout is not None else sys.stdout
)
@stdout.setter
def stdout(self, newout):
self.__class__._stdout = newout
def __init__(self, attributes=None, fgcolor=None, bgcolor=None, reset=False):
"""This is usually initialized from a factory."""
if isinstance(attributes, type(self)):
for item in ("attributes", "fg", "bg", "isreset"):
setattr(self, item, copy(getattr(attributes, item)))
return
self.attributes = attributes if attributes is not None else {}
self.fg = fgcolor
self.bg = bgcolor
self.isreset = reset
invalid_attributes = set(self.attributes) - set(self.attribute_names)
if len(invalid_attributes) > 0:
raise AttributeNotFound(
"Attribute(s) not valid: " + ", ".join(invalid_attributes)
)
@classmethod
def from_color(cls, color):
self = cls(fgcolor=color) if color.fg else cls(bgcolor=color)
return self
def invert(self):
"""This resets current color(s) and flips the value of all
attributes present"""
other = self.__class__()
# Opposite of reset is reset
if self.isreset:
other.isreset = True
return other
# Flip all attributes
for attribute in self.attributes:
other.attributes[attribute] = not self.attributes[attribute]
# Reset only if color present
if self.fg:
other.fg = self.fg.__class__()
if self.bg:
other.bg = self.bg.__class__()
return other
@property
def reset(self):
"""Shortcut to access reset as a property."""
return self.invert()
def __copy__(self):
"""Copy is supported, will make dictionary and colors unique."""
result = self.__class__()
result.isreset = self.isreset
result.fg = copy(self.fg)
result.bg = copy(self.bg)
result.attributes = copy(self.attributes)
return result
def __invert__(self):
"""This allows ~color."""
return self.invert()
def __add__(self, other):
"""Adding two matching Styles results in a new style with
the combination of both. Adding with a string results in
the string concatenation of a style.
Addition is non-commutative, with the rightmost Style property
being taken if both have the same property.
(Not safe)"""
if type(self) == type(other):
result = copy(other)
result.isreset = self.isreset or other.isreset
for attribute in self.attributes:
if attribute not in result.attributes:
result.attributes[attribute] = self.attributes[attribute]
if not result.fg:
result.fg = self.fg
if not result.bg:
result.bg = self.bg
return result
return other.__class__(self) + other
def __radd__(self, other):
"""This only gets called if the string is on the left side. (Not safe)"""
return other + other.__class__(self)
def wrap(self, wrap_this):
"""Wrap a string in this style and its inverse."""
return self + wrap_this + ~self
def __and__(self, other):
"""This class supports ``color & color2`` syntax,
and ``color & "String" syntax too.``"""
if type(self) == type(other):
return self + other
return self.wrap(other)
def __rand__(self, other):
"""This class supports ``"String:" & color`` syntax."""
return self.wrap(other)
def __ror__(self, other):
"""Support for "String" | color syntax"""
return self.wrap(other)
def __or__(self, other):
"""This class supports ``color | color2`` syntax. It also supports
``"color | "String"`` syntax too."""
return self.__and__(other)
def __call__(self):
"""\
This is a shortcut to print color immediately to the stdout. (Not safe)
"""
self.now()
def now(self):
"""Immediately writes color to stdout. (Not safe)"""
self.stdout.write(str(self))
def print(self, *printables, **kargs):
"""\
This acts like print; will print that argument to stdout wrapped
in Style with the same syntax as the print function in 3.4."""
end = kargs.get("end", self.end)
sep = kargs.get("sep", " ")
file = kargs.get("file", self.stdout)
flush = kargs.get("flush", False)
file.write(self.wrap(sep.join(map(str, printables))) + end)
if flush:
file.flush()
print_ = print
"""DEPRECATED: Shortcut from classic Python 2"""
def __getitem__(self, wrapped):
"""The [] syntax is supported for wrapping"""
return self.wrap(wrapped)
def __enter__(self):
"""Context manager support"""
self.stdout.write(str(self))
self.stdout.flush()
def __exit__(self, _type, _value, _traceback):
"""Runs even if exception occurred, does not catch it."""
self.stdout.write(str(~self))
self.stdout.flush()
return False
@property
def ansi_codes(self):
"""Generates the full ANSI code sequence for a Style"""
if self.isreset:
return [0]
codes = []
for attribute in self.attributes:
if self.attributes[attribute]:
codes.append(attributes_ansi[attribute])
else:
# Fixing bold inverse being 22 instead of 21 on some terminals:
codes.append(
attributes_ansi[attribute] + 20
if attributes_ansi[attribute] != 1
else 22
)
if self.fg:
codes.extend(self.fg.ansi_codes)
if self.bg:
self.bg.fg = False
codes.extend(self.bg.ansi_codes)
return codes
@property
def ansi_sequence(self):
"""This is the string ANSI sequence."""
codes = ";".join(str(c) for c in self.ansi_codes)
return f"\033[{codes}m" if codes else ""
def __repr__(self):
name = self.__class__.__name__
attributes = ", ".join(a for a in self.attributes if self.attributes[a])
neg_attributes = ", ".join(
f"-{a}" for a in self.attributes if not self.attributes[a]
)
colors = ", ".join(repr(c) for c in (self.fg, self.bg) if c)
string = (
"; ".join(s for s in (attributes, neg_attributes, colors) if s) or "empty"
)
if self.isreset:
string = "reset"
return f"<{name}: {string}>"
def __eq__(self, other):
"""Equality is true only if reset, or if attributes, fg, and bg match."""
if type(self) == type(other):
if self.isreset:
return other.isreset
return (
self.attributes == other.attributes
and self.fg == other.fg
and self.bg == other.bg
)
return str(self) == other
@abstractmethod
def __str__(self):
"""Base Style does not implement a __str__ representation. This is the one
required method of a subclass."""
@classmethod
def from_ansi(cls, ansi_string, filter_resets=False):
"""This generated a style from an ansi string. Will ignore resets if filter_resets is True."""
result = cls()
res = cls.ANSI_REG.search(ansi_string)
for group in res.groups():
sequence = map(int, group.split(";"))
result.add_ansi(sequence, filter_resets)
return result
def add_ansi(self, sequence, filter_resets=False):
"""Adds a sequence of ansi numbers to the class. Will ignore resets if filter_resets is True."""
values = iter(sequence)
try:
while True:
value = next(values)
if value in {38, 48}:
fg = value == 38
value = next(values)
if value == 5:
value = next(values)
if fg:
self.fg = self.color_class.from_full(value)
else:
self.bg = self.color_class.from_full(value, fg=False)
elif value == 2:
r = next(values)
g = next(values)
b = next(values)
if fg:
self.fg = self.color_class(r, g, b)
else:
self.bg = self.color_class(r, g, b, fg=False)
else:
raise ColorNotFound("the value 5 or 2 should follow a 38 or 48")
elif value == 0:
if filter_resets is False:
self.isreset = True
elif value in attributes_ansi.values():
for name, att_value in attributes_ansi.items():
if value == att_value:
self.attributes[name] = True
elif value in (20 + n for n in attributes_ansi.values()):
if filter_resets is False:
for name, att_value in attributes_ansi.items():
if value == att_value + 20:
self.attributes[name] = False
elif 30 <= value <= 37:
self.fg = self.color_class.from_simple(value - 30)
elif 40 <= value <= 47:
self.bg = self.color_class.from_simple(value - 40, fg=False)
elif 90 <= value <= 97:
self.fg = self.color_class.from_simple(value - 90 + 8)
elif 100 <= value <= 107:
self.bg = self.color_class.from_simple(value - 100 + 8, fg=False)
elif value == 39:
if filter_resets is False:
self.fg = self.color_class()
elif value == 49:
if filter_resets is False:
self.bg = self.color_class(fg=False)
else:
raise ColorNotFound(f"The code {value} is not recognised")
except StopIteration:
return
@classmethod
def string_filter_ansi(cls, colored_string):
"""Filters out colors in a string, returning only the name."""
return cls.ANSI_REG.sub("", colored_string)
@classmethod
def string_contains_colors(cls, colored_string):
"""Checks to see if a string contains colors."""
return len(cls.ANSI_REG.findall(colored_string)) > 0
def to_representation(self, rep):
"""This converts both colors to a specific representation"""
other = copy(self)
if other.fg:
other.fg = other.fg.to_representation(rep)
if other.bg:
other.bg = other.bg.to_representation(rep)
return other
def limit_representation(self, rep):
"""This only converts if true representation is higher"""
if rep is True or rep is False:
return self
other = copy(self)
if other.fg:
other.fg = other.fg.limit_representation(rep)
if other.bg:
other.bg = other.bg.limit_representation(rep)
return other
@property
def basic(self):
"""The color in the 8 color representation."""
return self.to_representation(1)
@property
def simple(self):
"""The color in the 16 color representation."""
return self.to_representation(2)
@property
def full(self):
"""The color in the 256 color representation."""
return self.to_representation(3)
@property
def true(self):
"""The color in the true color representation."""
return self.to_representation(4)
class ANSIStyle(Style):
"""This is a subclass for ANSI styles. Use it to get
color on sys.stdout tty terminals on posix systems.
Set ``use_color = True/False`` if you want to control color
for anything using this Style."""
__slots__ = ()
use_color = get_color_repr()
attribute_names = attributes_ansi
def __str__(self):
return (
self.limit_representation(self.use_color).ansi_sequence
if self.use_color
else ""
)
class HTMLStyle(Style):
"""This was meant to be a demo of subclassing Style, but
actually can be a handy way to quickly color html text."""
__slots__ = ()
attribute_names = dict(
bold="b",
em="em",
italics="i",
li="li",
underline='span style="text-decoration: underline;"',
code="code",
ol="ol start=0",
strikeout="s",
)
end = "<br/>\n"
def __str__(self):
if self.isreset:
raise ResetNotSupported("HTML does not support global resets!")
result = ""
if self.bg and not self.bg.isreset:
result += f'<span style="background-color: {self.bg.hex_code}">'
if self.fg and not self.fg.isreset:
result += f'<font color="{self.fg.hex_code}">'
for attr in sorted(self.attributes):
if self.attributes[attr]:
result += "<" + self.attribute_names[attr] + ">"
for attr in reversed(sorted(self.attributes)):
if not self.attributes[attr]:
result += "</" + self.attribute_names[attr].split(" ")[0] + ">"
if self.fg and self.fg.isreset:
result += "</font>"
if self.bg and self.bg.isreset:
result += "</span>"
return result

View File

@@ -0,0 +1,21 @@
"""
This module imitates a real module, providing standard syntax
like from `plumbum.colors` and from `plumbum.colors.bg` to work alongside
all the standard syntax for colors.
"""
import atexit
import sys
from plumbum.colorlib import ansicolors, main
_reset = ansicolors.reset.now
if __name__ == "__main__":
main()
else: # Don't register an exit if this is called using -m!
atexit.register(_reset)
sys.modules[__name__ + ".fg"] = ansicolors.fg
sys.modules[__name__ + ".bg"] = ansicolors.bg
sys.modules[__name__] = ansicolors # type: ignore[assignment]

View File

@@ -0,0 +1,49 @@
from plumbum.commands.base import (
ERROUT,
BaseCommand,
ConcreteCommand,
shquote,
shquote_list,
)
from plumbum.commands.modifiers import (
BG,
FG,
NOHUP,
RETCODE,
TEE,
TF,
ExecutionModifier,
Future,
)
from plumbum.commands.processes import (
CommandNotFound,
ProcessExecutionError,
ProcessLineTimedOut,
ProcessTimedOut,
run_proc,
)
__all__ = (
"BaseCommand",
"ConcreteCommand",
"shquote",
"shquote_list",
"ERROUT",
"BG",
"FG",
"NOHUP",
"RETCODE",
"TEE",
"TF",
"ExecutionModifier",
"Future",
"CommandNotFound",
"ProcessExecutionError",
"ProcessLineTimedOut",
"ProcessTimedOut",
"run_proc",
)
def __dir__():
return __all__

View File

@@ -0,0 +1,610 @@
import contextlib
import functools
import shlex
import subprocess
from subprocess import PIPE, Popen
from tempfile import TemporaryFile
from types import MethodType
from typing import ClassVar
import plumbum.commands.modifiers
from plumbum.commands.processes import iter_lines, run_proc
__all__ = (
"iter_lines",
"run_proc",
"shquote",
"shquote_list",
"RedirectionError",
"BaseCommand",
"Pipeline",
"BaseRedirection",
"BoundCommand",
"BoundEnvCommand",
"ConcreteCommand",
"ERROUT",
"StdinRedirection",
"StdoutRedirection",
"StderrRedirection",
"AppendingStdoutRedirection",
"StdinDataRedirection",
)
class RedirectionError(Exception):
"""Raised when an attempt is made to redirect an process' standard handle,
which was already redirected to/from a file"""
# ===================================================================================================
# Utilities
# ===================================================================================================
# modified from the stdlib pipes module for windows
_safechars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@%_-+=:,./"
_funnychars = '"`$\\'
def shquote(text):
"""Quotes the given text with shell escaping (assumes as syntax similar to ``sh``)"""
text = str(text)
return shlex.quote(text)
def shquote_list(seq):
return [shquote(item) for item in seq]
# ===================================================================================================
# Commands
# ===================================================================================================
class BaseCommand:
"""Base of all command objects"""
__slots__ = ("cwd", "env", "custom_encoding", "__weakref__")
def __str__(self):
return " ".join(self.formulate())
def __or__(self, other):
"""Creates a pipe with the other command"""
return Pipeline(self, other)
def __gt__(self, file):
"""Redirects the process' stdout to the given file"""
return StdoutRedirection(self, file)
def __rshift__(self, file):
"""Redirects the process' stdout to the given file (appending)"""
return AppendingStdoutRedirection(self, file)
def __ge__(self, file):
"""Redirects the process' stderr to the given file"""
return StderrRedirection(self, file)
def __lt__(self, file):
"""Redirects the given file into the process' stdin"""
return StdinRedirection(self, file)
def __lshift__(self, data):
"""Redirects the given data into the process' stdin"""
return StdinDataRedirection(self, data)
def __getitem__(self, args):
"""Creates a bound-command with the given arguments. Shortcut for
bound_command."""
if not isinstance(args, (tuple, list)):
args = [
args,
]
return self.bound_command(*args)
def bound_command(self, *args):
"""Creates a bound-command with the given arguments"""
if not args:
return self
if isinstance(self, BoundCommand):
return BoundCommand(self.cmd, self.args + list(args))
return BoundCommand(self, args)
def __call__(self, *args, **kwargs):
"""A shortcut for `run(args)`, returning only the process' stdout"""
return self.run(args, **kwargs)[1]
def _get_encoding(self):
raise NotImplementedError()
def with_env(self, **env):
"""Returns a BoundEnvCommand with the given environment variables"""
if not env:
return self
return BoundEnvCommand(self, env=env)
def with_cwd(self, path):
"""
Returns a BoundEnvCommand with the specified working directory.
This overrides a cwd specified in a wrapping `machine.cwd()` context manager.
"""
if not path:
return self
return BoundEnvCommand(self, cwd=path)
setenv = with_env
@property
def machine(self):
raise NotImplementedError()
def formulate(self, level=0, args=()):
"""Formulates the command into a command-line, i.e., a list of shell-quoted strings
that can be executed by ``Popen`` or shells.
:param level: The nesting level of the formulation; it dictates how much shell-quoting
(if any) should be performed
:param args: The arguments passed to this command (a tuple)
:returns: A list of strings
"""
raise NotImplementedError()
def popen(self, args=(), **kwargs):
"""Spawns the given command, returning a ``Popen``-like object.
.. note::
When processes run in the **background** (either via ``popen`` or
:class:`& BG <plumbum.commands.BG>`), their stdout/stderr pipes might fill up,
causing them to hang. If you know a process produces output, be sure to consume it
every once in a while, using a monitoring thread/reactor in the background.
For more info, see `#48 <https://github.com/tomerfiliba/plumbum/issues/48>`_
:param args: Any arguments to be passed to the process (a tuple)
:param kwargs: Any keyword-arguments to be passed to the ``Popen`` constructor
:returns: A ``Popen``-like object
"""
raise NotImplementedError()
def nohup(self, cwd=".", stdout="nohup.out", stderr=None, append=True):
"""Runs a command detached."""
return self.machine.daemonic_popen(self, cwd, stdout, stderr, append)
@contextlib.contextmanager
def bgrun(self, args=(), **kwargs):
"""Runs the given command as a context manager, allowing you to create a
`pipeline <http://en.wikipedia.org/wiki/Pipeline_(computing)>`_ (not in the UNIX sense)
of programs, parallelizing their work. In other words, instead of running programs
one after the other, you can start all of them at the same time and wait for them to
finish. For a more thorough review, see
`Lightweight Asynchronism <http://tomerfiliba.com/blog/Toying-with-Context-Managers/>`_.
Example::
from plumbum.cmd import mkfs
with mkfs["-t", "ext3", "/dev/sda1"] as p1:
with mkfs["-t", "ext3", "/dev/sdb1"] as p2:
pass
.. note::
When processes run in the **background** (either via ``popen`` or
:class:`& BG <plumbum.commands.BG>`), their stdout/stderr pipes might fill up,
causing them to hang. If you know a process produces output, be sure to consume it
every once in a while, using a monitoring thread/reactor in the background.
For more info, see `#48 <https://github.com/tomerfiliba/plumbum/issues/48>`_
For the arguments, see :func:`run <BaseCommand.run>`.
:returns: A Popen object, augmented with a ``.run()`` method, which returns a tuple of
(return code, stdout, stderr)
"""
retcode = kwargs.pop("retcode", 0)
timeout = kwargs.pop("timeout", None)
p = self.popen(args, **kwargs)
was_run = [False]
def runner():
if was_run[0]:
return None # already done
was_run[0] = True
try:
return run_proc(p, retcode, timeout)
finally:
del p.run # to break cyclic reference p -> cell -> p
for f in (p.stdin, p.stdout, p.stderr):
with contextlib.suppress(Exception):
f.close()
p.run = runner
yield p
runner()
def run(self, args=(), **kwargs):
"""Runs the given command (equivalent to popen() followed by
:func:`run_proc <plumbum.commands.run_proc>`). If the exit code of the process does
not match the expected one, :class:`ProcessExecutionError
<plumbum.commands.ProcessExecutionError>` is raised.
:param args: Any arguments to be passed to the process (a tuple)
:param retcode: The expected return code of this process (defaults to 0).
In order to disable exit-code validation, pass ``None``. It may also
be a tuple (or any iterable) of expected exit codes.
.. note:: this argument must be passed as a keyword argument.
:param timeout: The maximal amount of time (in seconds) to allow the process to run.
``None`` means no timeout is imposed; otherwise, if the process hasn't
terminated after that many seconds, the process will be forcefully
terminated an exception will be raised
.. note:: this argument must be passed as a keyword argument.
:param kwargs: Any keyword-arguments to be passed to the ``Popen`` constructor
:returns: A tuple of (return code, stdout, stderr)
"""
with self.bgrun(args, **kwargs) as p:
return p.run()
def _use_modifier(self, modifier, args):
"""
Applies a modifier to the current object (e.g. FG, NOHUP)
:param modifier: The modifier class to apply (e.g. FG)
:param args: A dictionary of arguments to pass to this modifier
:return:
"""
modifier_instance = modifier(**args)
return self & modifier_instance
def run_bg(self, **kwargs):
"""
Run this command in the background. Uses all arguments from the BG construct
:py:class: `plumbum.commands.modifiers.BG`
"""
return self._use_modifier(plumbum.commands.modifiers.BG, kwargs)
def run_fg(self, **kwargs):
"""
Run this command in the foreground. Uses all arguments from the FG construct
:py:class: `plumbum.commands.modifiers.FG`
"""
return self._use_modifier(plumbum.commands.modifiers.FG, kwargs)
def run_tee(self, **kwargs):
"""
Run this command using the TEE construct. Inherits all arguments from TEE
:py:class: `plumbum.commands.modifiers.TEE`
"""
return self._use_modifier(plumbum.commands.modifiers.TEE, kwargs)
def run_tf(self, **kwargs):
"""
Run this command using the TF construct. Inherits all arguments from TF
:py:class: `plumbum.commands.modifiers.TF`
"""
return self._use_modifier(plumbum.commands.modifiers.TF, kwargs)
def run_retcode(self, **kwargs):
"""
Run this command using the RETCODE construct. Inherits all arguments from RETCODE
:py:class: `plumbum.commands.modifiers.RETCODE`
"""
return self._use_modifier(plumbum.commands.modifiers.RETCODE, kwargs)
def run_nohup(self, **kwargs):
"""
Run this command using the NOHUP construct. Inherits all arguments from NOHUP
:py:class: `plumbum.commands.modifiers.NOHUP`
"""
return self._use_modifier(plumbum.commands.modifiers.NOHUP, kwargs)
class BoundCommand(BaseCommand):
__slots__ = ("cmd", "args")
def __init__(self, cmd, args):
self.cmd = cmd
self.args = list(args)
def __repr__(self):
return f"BoundCommand({self.cmd!r}, {self.args!r})"
def _get_encoding(self):
return self.cmd._get_encoding()
def formulate(self, level=0, args=()):
return self.cmd.formulate(level + 1, self.args + list(args))
@property
def machine(self):
return self.cmd.machine
def popen(self, args=(), **kwargs):
if isinstance(args, str):
args = [
args,
]
return self.cmd.popen(self.args + list(args), **kwargs)
class BoundEnvCommand(BaseCommand):
__slots__ = ("cmd",)
def __init__(self, cmd, env=None, cwd=None):
self.cmd = cmd
self.env = env or {}
self.cwd = cwd
def __repr__(self):
return f"BoundEnvCommand({self.cmd!r}, {self.env!r})"
def _get_encoding(self):
return self.cmd._get_encoding()
def formulate(self, level=0, args=()):
return self.cmd.formulate(level, args)
@property
def machine(self):
return self.cmd.machine
def popen(self, args=(), cwd=None, env=None, **kwargs):
env = env or {}
return self.cmd.popen(
args,
cwd=self.cwd if cwd is None else cwd,
env=dict(self.env, **env),
**kwargs,
)
class Pipeline(BaseCommand):
__slots__ = ("srccmd", "dstcmd")
def __init__(self, srccmd, dstcmd):
self.srccmd = srccmd
self.dstcmd = dstcmd
def __repr__(self):
return f"Pipeline({self.srccmd!r}, {self.dstcmd!r})"
def _get_encoding(self):
return self.srccmd._get_encoding() or self.dstcmd._get_encoding()
def formulate(self, level=0, args=()):
return (
self.srccmd.formulate(level + 1)
+ ["|"]
+ self.dstcmd.formulate(level + 1, args)
)
@property
def machine(self):
return self.srccmd.machine
def popen(self, args=(), **kwargs):
src_kwargs = kwargs.copy()
src_kwargs["stdout"] = PIPE
if "stdin" in kwargs:
src_kwargs["stdin"] = kwargs["stdin"]
srcproc = self.srccmd.popen(args, **src_kwargs)
kwargs["stdin"] = srcproc.stdout
dstproc = self.dstcmd.popen(**kwargs)
# allow p1 to receive a SIGPIPE if p2 exits
srcproc.stdout.close()
if srcproc.stderr is not None:
dstproc.stderr = srcproc.stderr
if srcproc.stdin and src_kwargs.get("stdin") != PIPE:
srcproc.stdin.close()
dstproc.srcproc = srcproc
# monkey-patch .wait() to wait on srcproc as well (it's expected to die when dstproc dies)
dstproc_wait = dstproc.wait
@functools.wraps(Popen.wait)
def wait2(*args, **kwargs):
rc_dst = dstproc_wait(*args, **kwargs)
rc_src = srcproc.wait(*args, **kwargs)
dstproc.returncode = rc_dst or rc_src
return dstproc.returncode
dstproc._proc.wait = wait2
dstproc_verify = dstproc.verify
def verify(proc, retcode, timeout, stdout, stderr):
# TODO: right now it's impossible to specify different expected
# return codes for different stages of the pipeline
try:
or_retcode = [0] + list(retcode)
except TypeError:
if retcode is None:
or_retcode = None # no-retcode-verification acts "greedily"
else:
or_retcode = [0, retcode]
proc.srcproc.verify(or_retcode, timeout, stdout, stderr)
dstproc_verify(retcode, timeout, stdout, stderr)
dstproc.verify = MethodType(verify, dstproc)
dstproc.stdin = srcproc.stdin
return dstproc
class BaseRedirection(BaseCommand):
__slots__ = ("cmd", "file")
SYM: ClassVar[str]
KWARG: ClassVar[str]
MODE: ClassVar[str]
def __init__(self, cmd, file):
self.cmd = cmd
self.file = file
def _get_encoding(self):
return self.cmd._get_encoding()
def __repr__(self):
return f"{self.__class__.__name__}({self.cmd!r}, {self.file!r})"
def formulate(self, level=0, args=()):
return self.cmd.formulate(level + 1, args) + [
self.SYM,
shquote(getattr(self.file, "name", self.file)),
]
@property
def machine(self):
return self.cmd.machine
def popen(self, args=(), **kwargs):
from plumbum.machines.local import LocalPath
from plumbum.machines.remote import RemotePath
if self.KWARG in kwargs and kwargs[self.KWARG] not in (PIPE, None):
raise RedirectionError(f"{self.KWARG} is already redirected")
if isinstance(self.file, RemotePath):
raise TypeError("Cannot redirect to/from remote paths")
if isinstance(self.file, (str, LocalPath)):
f = kwargs[self.KWARG] = open(str(self.file), self.MODE, encoding="utf-8")
else:
kwargs[self.KWARG] = self.file
f = None
try:
return self.cmd.popen(args, **kwargs)
finally:
if f:
f.close()
class StdinRedirection(BaseRedirection):
__slots__ = ()
SYM = "<"
KWARG = "stdin"
MODE = "r"
class StdoutRedirection(BaseRedirection):
__slots__ = ()
SYM = ">"
KWARG = "stdout"
MODE = "w"
class AppendingStdoutRedirection(BaseRedirection):
__slots__ = ()
SYM = ">>"
KWARG = "stdout"
MODE = "a"
class StderrRedirection(BaseRedirection):
__slots__ = ()
SYM = "2>"
KWARG = "stderr"
MODE = "w"
class _ERROUT(int):
def __repr__(self):
return "ERROUT"
def __str__(self):
return "&1"
ERROUT = _ERROUT(subprocess.STDOUT)
class StdinDataRedirection(BaseCommand):
__slots__ = ("cmd", "data")
CHUNK_SIZE = 16000
def __init__(self, cmd, data):
self.cmd = cmd
self.data = data
def _get_encoding(self):
return self.cmd._get_encoding()
def formulate(self, level=0, args=()):
return [
f"echo {shquote(self.data)}",
"|",
self.cmd.formulate(level + 1, args),
]
@property
def machine(self):
return self.cmd.machine
def popen(self, args=(), **kwargs):
if kwargs.get("stdin") not in (PIPE, None):
raise RedirectionError("stdin is already redirected")
data = self.data
if isinstance(data, str) and self._get_encoding() is not None:
data = data.encode(self._get_encoding())
f = TemporaryFile()
while data:
chunk = data[: self.CHUNK_SIZE]
f.write(chunk)
data = data[self.CHUNK_SIZE :]
f.seek(0)
kwargs["stdin"] = f
# try:
return self.cmd.popen(args, **kwargs)
# finally:
# f.close()
class ConcreteCommand(BaseCommand):
QUOTE_LEVEL: ClassVar[int]
__slots__ = ("executable",)
def __init__(self, executable, encoding):
self.executable = executable
self.custom_encoding = encoding
self.cwd = None
self.env = None
def __str__(self):
return str(self.executable)
def __repr__(self):
return f"{type(self).__name__}({self.executable})"
def _get_encoding(self):
return self.custom_encoding
def formulate(self, level=0, args=()):
argv = [str(self.executable)]
for a in args:
if a is None:
continue
if isinstance(a, BaseCommand):
if level >= self.QUOTE_LEVEL:
argv.extend(shquote_list(a.formulate(level + 1)))
else:
argv.extend(a.formulate(level + 1))
elif isinstance(a, (list, tuple)):
argv.extend(
shquote(b) if level >= self.QUOTE_LEVEL else str(b) for b in a
)
else:
argv.append(shquote(a) if level >= self.QUOTE_LEVEL else str(a))
# if self.custom_encoding:
# argv = [a.encode(self.custom_encoding) for a in argv if isinstance(a, str)]
return argv
@property
def machine(self):
raise NotImplementedError()
def popen(self, args=(), **kwargs):
raise NotImplementedError()

View File

@@ -0,0 +1,125 @@
import contextlib
import errno
import os
import signal
import subprocess
import sys
import time
import traceback
from plumbum.commands.processes import ProcessExecutionError
class _fake_lock:
"""Needed to allow normal os.exit() to work without error"""
@staticmethod
def acquire(_):
return True
@staticmethod
def release():
pass
def posix_daemonize(command, cwd, stdout=None, stderr=None, append=True):
if stdout is None:
stdout = os.devnull
if stderr is None:
stderr = stdout
MAX_SIZE = 16384
rfd, wfd = os.pipe()
argv = command.formulate()
firstpid = os.fork()
if firstpid == 0:
# first child: become session leader
os.close(rfd)
rc = 0
try:
os.setsid()
os.umask(0)
stdin = open(os.devnull, encoding="utf-8")
stdout = open(stdout, "a" if append else "w", encoding="utf-8")
stderr = open(stderr, "a" if append else "w", encoding="utf-8")
signal.signal(signal.SIGHUP, signal.SIG_IGN)
proc = command.popen(
cwd=cwd,
close_fds=True,
stdin=stdin.fileno(),
stdout=stdout.fileno(),
stderr=stderr.fileno(),
)
os.write(wfd, str(proc.pid).encode("utf8"))
except Exception:
rc = 1
tbtext = "".join(traceback.format_exception(*sys.exc_info()))[-MAX_SIZE:]
os.write(wfd, tbtext.encode("utf8"))
finally:
os.close(wfd)
os._exit(rc)
# wait for first child to die
os.close(wfd)
_, rc = os.waitpid(firstpid, 0)
output = os.read(rfd, MAX_SIZE)
os.close(rfd)
with contextlib.suppress(UnicodeError):
output = output.decode("utf8")
if rc == 0 and output.isdigit():
secondpid = int(output)
else:
raise ProcessExecutionError(argv, rc, "", output)
proc = subprocess.Popen.__new__(subprocess.Popen)
proc._child_created = True
proc.returncode = None
proc.stdout = None
proc.stdin = None
proc.stderr = None
proc.pid = secondpid
proc.universal_newlines = False
proc._input = None
proc._waitpid_lock = _fake_lock()
proc._communication_started = False
proc.args = argv
proc.argv = argv
def poll(self=proc):
if self.returncode is None:
try:
os.kill(self.pid, 0)
except OSError as ex:
if ex.errno == errno.ESRCH:
# process does not exist
self.returncode = 0
else:
raise
return self.returncode
def wait(self=proc):
while self.returncode is None:
if self.poll() is None:
time.sleep(0.5)
return proc.returncode
proc.poll = poll
proc.wait = wait
return proc
def win32_daemonize(command, cwd, stdout=None, stderr=None, append=True):
if stdout is None:
stdout = os.devnull
if stderr is None:
stderr = stdout
DETACHED_PROCESS = 0x00000008
stdin = open(os.devnull, encoding="utf-8")
stdout = open(stdout, "a" if append else "w", encoding="utf-8")
stderr = open(stderr, "a" if append else "w", encoding="utf-8")
return command.popen(
cwd=cwd,
stdin=stdin.fileno(),
stdout=stdout.fileno(),
stderr=stderr.fileno(),
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS,
)

View File

@@ -0,0 +1,503 @@
import sys
from logging import DEBUG, INFO
from select import select
from subprocess import PIPE
import plumbum.commands.base
from plumbum.commands.processes import BY_TYPE, ProcessExecutionError, run_proc
from plumbum.lib import read_fd_decode_safely
class Future:
"""Represents a "future result" of a running process. It basically wraps a ``Popen``
object and the expected exit code, and provides poll(), wait(), returncode, stdout,
and stderr.
"""
def __init__(self, proc, expected_retcode, timeout=None):
self.proc = proc
self._expected_retcode = expected_retcode
self._timeout = timeout
self._returncode = None
self._stdout = None
self._stderr = None
def __repr__(self):
running = self._returncode if self.ready() else "running"
return f"<Future {self.proc.argv!r} ({running})>"
def poll(self):
"""Polls the underlying process for termination; returns ``False`` if still running,
or ``True`` if terminated"""
if self.proc.poll() is not None:
self.wait()
return self._returncode is not None
ready = poll
def wait(self):
"""Waits for the process to terminate; will raise a
:class:`plumbum.commands.ProcessExecutionError` in case of failure"""
if self._returncode is not None:
return
self._returncode, self._stdout, self._stderr = run_proc(
self.proc, self._expected_retcode, self._timeout
)
@property
def stdout(self):
"""The process' stdout; accessing this property will wait for the process to finish"""
self.wait()
return self._stdout
@property
def stderr(self):
"""The process' stderr; accessing this property will wait for the process to finish"""
self.wait()
return self._stderr
@property
def returncode(self):
"""The process' returncode; accessing this property will wait for the process to finish"""
self.wait()
return self._returncode
# ===================================================================================================
# execution modifiers
# ===================================================================================================
class ExecutionModifier:
__slots__ = ("__weakref__",)
def __repr__(self):
"""Automatically creates a representation for given subclass with slots.
Ignore hidden properties."""
slots = {}
for cls in self.__class__.__mro__:
slots_list = getattr(cls, "__slots__", ())
if isinstance(slots_list, str):
slots_list = (slots_list,)
for prop in slots_list:
if prop[0] != "_":
slots[prop] = getattr(self, prop)
mystrs = (f"{name} = {value}" for name, value in slots.items())
mystrs_str = ", ".join(mystrs)
return f"{self.__class__.__name__}({mystrs_str})"
@classmethod
def __call__(cls, *args, **kwargs):
return cls(*args, **kwargs)
class _BG(ExecutionModifier):
"""
An execution modifier that runs the given command in the background, returning a
:class:`Future <plumbum.commands.Future>` object. In order to mimic shell syntax, it applies
when you right-and it with a command. If you wish to expect a different return code
(other than the normal success indicate by 0), use ``BG(retcode)``. Example::
future = sleep[5] & BG # a future expecting an exit code of 0
future = sleep[5] & BG(7) # a future expecting an exit code of 7
.. note::
When processes run in the **background** (either via ``popen`` or
:class:`& BG <plumbum.commands.BG>`), their stdout/stderr pipes might fill up,
causing them to hang. If you know a process produces output, be sure to consume it
every once in a while, using a monitoring thread/reactor in the background.
For more info, see `#48 <https://github.com/tomerfiliba/plumbum/issues/48>`_
"""
__slots__ = ("retcode", "kargs", "timeout")
def __init__(self, retcode=0, timeout=None, **kargs):
self.retcode = retcode
self.kargs = kargs
self.timeout = timeout
def __rand__(self, cmd):
return Future(cmd.popen(**self.kargs), self.retcode, timeout=self.timeout)
class _FG(ExecutionModifier):
"""
An execution modifier that runs the given command in the foreground, passing it the
current process' stdin, stdout and stderr. Useful for interactive programs that require
a TTY. There is no return value.
In order to mimic shell syntax, it applies when you right-and it with a command.
If you wish to expect a different return code (other than the normal success indicate by 0),
use ``FG(retcode)``. Example::
vim & FG # run vim in the foreground, expecting an exit code of 0
vim & FG(7) # run vim in the foreground, expecting an exit code of 7
"""
__slots__ = ("retcode", "timeout")
def __init__(self, retcode=0, timeout=None):
self.retcode = retcode
self.timeout = timeout
def __rand__(self, cmd):
cmd(
retcode=self.retcode,
stdin=None,
stdout=None,
stderr=None,
timeout=self.timeout,
)
class _TEE(ExecutionModifier):
"""Run a command, dumping its stdout/stderr to the current process's stdout
and stderr, but ALSO return them. Useful for interactive programs that
expect a TTY but also have valuable output.
Use as:
ls["-l"] & TEE
Returns a tuple of (return code, stdout, stderr), just like ``run()``.
"""
__slots__ = ("retcode", "buffered", "timeout")
def __init__(self, retcode=0, buffered=True, timeout=None):
"""`retcode` is the return code to expect to mean "success". Set
`buffered` to False to disable line-buffering the output, which may
cause stdout and stderr to become more entangled than usual.
"""
self.retcode = retcode
self.buffered = buffered
self.timeout = timeout
def __rand__(self, cmd):
with cmd.bgrun(
retcode=self.retcode,
stdin=None,
stdout=PIPE,
stderr=PIPE,
timeout=self.timeout,
) as p:
outbuf = []
errbuf = []
out = p.stdout
err = p.stderr
buffers = {out: outbuf, err: errbuf}
tee_to = {out: sys.stdout, err: sys.stderr}
done = False
while not done:
# After the process exits, we have to do one more
# round of reading in order to drain any data in the
# pipe buffer. Thus, we check poll() here,
# unconditionally enter the read loop, and only then
# break out of the outer loop if the process has
# exited.
done = p.poll() is not None
# We continue this loop until we've done a full
# `select()` call without collecting any input. This
# ensures that our final pass -- after process exit --
# actually drains the pipe buffers, even if it takes
# multiple calls to read().
progress = True
while progress:
progress = False
ready, _, _ = select((out, err), (), ())
for fd in ready:
buf = buffers[fd]
data, text = read_fd_decode_safely(fd, 4096)
if not data: # eof
continue
progress = True
# Python conveniently line-buffers stdout and stderr for
# us, so all we need to do is write to them
# This will automatically add up to three bytes if it cannot be decoded
tee_to[fd].write(text)
# And then "unbuffered" is just flushing after each write
if not self.buffered:
tee_to[fd].flush()
buf.append(data)
stdout = "".join([x.decode("utf-8") for x in outbuf])
stderr = "".join([x.decode("utf-8") for x in errbuf])
return p.returncode, stdout, stderr
class _TF(ExecutionModifier):
"""
An execution modifier that runs the given command, but returns True/False depending on the retcode.
This returns True if the expected exit code is returned, and false if it is not.
This is useful for checking true/false bash commands.
If you wish to expect a different return code (other than the normal success indicate by 0),
use ``TF(retcode)``. If you want to run the process in the foreground, then use
``TF(FG=True)``.
Example::
local['touch']['/root/test'] & TF * Returns False, since this cannot be touched
local['touch']['/root/test'] & TF(1) # Returns True
local['touch']['/root/test'] & TF(FG=True) * Returns False, will show error message
"""
__slots__ = ("retcode", "FG", "timeout")
def __init__(
self,
retcode=0,
FG=False, # pylint: disable=redefined-outer-name
timeout=None,
):
"""`retcode` is the return code to expect to mean "success". Set
`FG` to True to run in the foreground.
"""
self.retcode = retcode
self.FG = FG
self.timeout = timeout
@classmethod
def __call__(cls, *args, **kwargs):
return cls(*args, **kwargs)
def __rand__(self, cmd):
try:
if self.FG:
cmd(
retcode=self.retcode,
stdin=None,
stdout=None,
stderr=None,
timeout=self.timeout,
)
else:
cmd(retcode=self.retcode, timeout=self.timeout)
return True
except ProcessExecutionError:
return False
class _RETCODE(ExecutionModifier):
"""
An execution modifier that runs the given command, causing it to run and return the retcode.
This is useful for working with bash commands that have important retcodes but not very
useful output.
If you want to run the process in the foreground, then use ``RETCODE(FG=True)``.
Example::
local['touch']['/root/test'] & RETCODE # Returns 1, since this cannot be touched
local['touch']['/root/test'] & RETCODE(FG=True) * Returns 1, will show error message
"""
__slots__ = ("foreground", "timeout")
def __init__(
self,
FG=False, # pylint: disable=redefined-outer-name
timeout=None,
):
"""`FG` to True to run in the foreground."""
self.foreground = FG
self.timeout = timeout
@classmethod
def __call__(cls, *args, **kwargs):
return cls(*args, **kwargs)
def __rand__(self, cmd):
if self.foreground:
result = cmd.run(
retcode=None, stdin=None, stdout=None, stderr=None, timeout=self.timeout
)
return result[0]
return cmd.run(retcode=None, timeout=self.timeout)[0]
class _NOHUP(ExecutionModifier):
"""
An execution modifier that runs the given command in the background, disconnected
from the current process, returning a
standard popen object. It will keep running even if you close the current process.
In order to slightly mimic shell syntax, it applies
when you right-and it with a command. If you wish to use a diffent working directory
or different stdout, stderr, you can use named arguments. The default is ``NOHUP(
cwd=local.cwd, stdout='nohup.out', stderr=None)``. If stderr is None, stderr will be
sent to stdout. Use ``os.devnull`` for null output. Will respect redirected output.
Example::
sleep[5] & NOHUP # Outputs to nohup.out
sleep[5] & NOHUP(stdout=os.devnull) # No output
The equivalent bash command would be
.. code-block:: bash
nohup sleep 5 &
"""
__slots__ = ("cwd", "stdout", "stderr", "append")
def __init__(self, cwd=".", stdout="nohup.out", stderr=None, append=True):
"""Set ``cwd``, ``stdout``, or ``stderr``.
Runs as a forked process. You can set ``append=False``, too.
"""
self.cwd = cwd
self.stdout = stdout
self.stderr = stderr
self.append = append
def __rand__(self, cmd):
if isinstance(cmd, plumbum.commands.base.StdoutRedirection):
stdout = cmd.file
append = False
cmd = cmd.cmd
elif isinstance(cmd, plumbum.commands.base.AppendingStdoutRedirection):
stdout = cmd.file
append = True
cmd = cmd.cmd
else:
stdout = self.stdout
append = self.append
return cmd.nohup(self.cwd, stdout, self.stderr, append)
class LogPipe:
def __init__(self, line_timeout, kw, levels, prefix, log):
self.line_timeout = line_timeout
self.kw = kw
self.levels = levels
self.prefix = prefix
self.log = log
def __rand__(self, cmd):
popen = cmd if hasattr(cmd, "iter_lines") else cmd.popen()
for typ, lines in popen.iter_lines(
line_timeout=self.line_timeout, mode=BY_TYPE, **self.kw
):
if not lines:
continue
level = self.levels[typ]
for line in lines.splitlines():
if self.prefix:
line = f"{self.prefix}: {line}"
self.log(level, line)
return popen.returncode
class PipeToLoggerMixin:
"""
This mixin allows piping plumbum commands' output into a logger.
The logger must implement a ``log(level, msg)`` method, as in ``logging.Logger``
Example::
class MyLogger(logging.Logger, PipeToLoggerMixin):
pass
logger = MyLogger("example.app")
Here we send the output of an install.sh script into our log::
local['./install.sh'] & logger
We can choose the log-level for each stream::
local['./install.sh'] & logger.pipe(out_level=logging.DEBUG, err_level=logging.DEBUG)
Or use a convenience method for it::
local['./install.sh'] & logger.pipe_debug()
A prefix can be added to each line::
local['./install.sh'] & logger.pipe(prefix="install.sh: ")
If the command fails, an exception is raised as usual. This can be modified::
local['install.sh'] & logger.pipe_debug(retcode=None)
An exception is also raised if too much time (``DEFAULT_LINE_TIMEOUT``) passed between lines in the stream,
This can also be modified::
local['install.sh'] & logger.pipe(line_timeout=10)
If we happen to use logbook::
class MyLogger(logbook.Logger, PipeToLoggerMixin):
from logbook import DEBUG, INFO # hook up with logbook's levels
"""
DEFAULT_LINE_TIMEOUT = 10 * 60
DEFAULT_STDOUT = "INFO"
DEFAULT_STDERR = "DEBUG"
INFO = INFO
DEBUG = DEBUG
def pipe(
self, out_level=None, err_level=None, prefix=None, line_timeout=None, **kw
):
"""
Pipe a command's stdout and stderr lines into this logger.
:param out_level: the log level for lines coming from stdout
:param err_level: the log level for lines coming from stderr
Optionally use `prefix` for each line.
"""
levels = {
1: getattr(self, self.DEFAULT_STDOUT),
2: getattr(self, self.DEFAULT_STDERR),
}
if line_timeout is None:
line_timeout = self.DEFAULT_LINE_TIMEOUT
if out_level is not None:
levels[1] = out_level
if err_level is not None:
levels[2] = err_level
return LogPipe(line_timeout, kw, levels, prefix, self.log)
def pipe_info(self, prefix=None, **kw):
"""
Pipe a command's stdout and stderr lines into this logger (both at level INFO)
"""
return self.pipe(self.INFO, self.INFO, prefix=prefix, **kw)
def pipe_debug(self, prefix=None, **kw):
"""
Pipe a command's stdout and stderr lines into this logger (both at level DEBUG)
"""
return self.pipe(self.DEBUG, self.DEBUG, prefix=prefix, **kw)
def __rand__(self, cmd):
"""
Pipe a command's stdout and stderr lines into this logger.
Log levels for each stream are determined by ``DEFAULT_STDOUT`` and ``DEFAULT_STDERR``.
"""
return cmd & self.pipe(
getattr(self, self.DEFAULT_STDOUT), getattr(self, self.DEFAULT_STDERR)
)
BG = _BG()
FG = _FG()
NOHUP = _NOHUP()
RETCODE = _RETCODE()
TEE = _TEE()
TF = _TF()

View File

@@ -0,0 +1,391 @@
import atexit
import contextlib
import heapq
import math
import time
from queue import Empty as QueueEmpty
from queue import Queue
from threading import Thread
from plumbum.lib import IS_WIN32
# ===================================================================================================
# utility functions
# ===================================================================================================
def _check_process(proc, retcode, timeout, stdout, stderr):
proc.verify(retcode, timeout, stdout, stderr)
return proc.returncode, stdout, stderr
def _iter_lines_posix(proc, decode, linesize, line_timeout=None):
from selectors import EVENT_READ, DefaultSelector
# Python 3.4+ implementation
def selector():
sel = DefaultSelector()
sel.register(proc.stdout, EVENT_READ, 0)
sel.register(proc.stderr, EVENT_READ, 1)
while True:
ready = sel.select(line_timeout)
if not ready and line_timeout:
raise ProcessLineTimedOut(
"popen line timeout expired",
getattr(proc, "argv", None),
getattr(proc, "machine", None),
)
for key, _mask in ready:
yield key.data, decode(key.fileobj.readline(linesize))
for ret in selector():
yield ret
if proc.poll() is not None:
break
for line in proc.stdout:
yield 0, decode(line)
for line in proc.stderr:
yield 1, decode(line)
def _iter_lines_win32(proc, decode, linesize, line_timeout=None):
class Piper(Thread):
def __init__(self, fd, pipe):
super().__init__(name=f"PlumbumPiper{fd}Thread")
self.pipe = pipe
self.fd = fd
self.empty = False
self.daemon = True
super().start()
def read_from_pipe(self):
return self.pipe.readline(linesize)
def run(self):
for line in iter(self.read_from_pipe, b""):
queue.put((self.fd, decode(line)))
# self.pipe.close()
if line_timeout is None:
line_timeout = float("inf")
queue = Queue()
pipers = [Piper(0, proc.stdout), Piper(1, proc.stderr)]
last_line_ts = time.time()
empty = True
while True:
try:
yield queue.get_nowait()
last_line_ts = time.time()
empty = False
except QueueEmpty:
empty = True
if time.time() - last_line_ts > line_timeout:
raise ProcessLineTimedOut(
"popen line timeout expired",
getattr(proc, "argv", None),
getattr(proc, "machine", None),
)
if proc.poll() is not None:
break
if empty:
time.sleep(0.1)
for piper in pipers:
piper.join()
while True:
try:
yield queue.get_nowait()
except QueueEmpty:
break
if IS_WIN32:
_iter_lines = _iter_lines_win32
else:
_iter_lines = _iter_lines_posix
# ===================================================================================================
# Exceptions
# ===================================================================================================
class ProcessExecutionError(OSError):
"""Represents the failure of a process. When the exit code of a terminated process does not
match the expected result, this exception is raised by :func:`run_proc
<plumbum.commands.run_proc>`. It contains the process' return code, stdout, and stderr, as
well as the command line used to create the process (``argv``)
"""
# pylint: disable-next=super-init-not-called
def __init__(self, argv, retcode, stdout, stderr, message=None, *, host=None):
# we can't use 'super' here since OSError only keeps the first 2 args,
# which leads to failuring in loading this object from a pickle.dumps.
# pylint: disable-next=non-parent-init-called
Exception.__init__(self, argv, retcode, stdout, stderr)
self.message = message
self.host = host
self.argv = argv
self.retcode = retcode
if isinstance(stdout, bytes):
stdout = ascii(stdout)
if isinstance(stderr, bytes):
stderr = ascii(stderr)
self.stdout = stdout
self.stderr = stderr
def __str__(self):
# avoid an import cycle
from plumbum.commands.base import shquote_list
stdout = "\n | ".join(str(self.stdout).splitlines())
stderr = "\n | ".join(str(self.stderr).splitlines())
cmd = " ".join(shquote_list(self.argv))
lines = []
if self.message:
lines = [self.message, "\nReturn code: | ", str(self.retcode)]
else:
lines = ["Unexpected exit code: ", str(self.retcode)]
cmd = "\n | ".join(cmd.splitlines())
lines += ["\nCommand line: | ", cmd]
if self.host:
lines += ["\nHost: | ", self.host]
if stdout:
lines += ["\nStdout: | ", stdout]
if stderr:
lines += ["\nStderr: | ", stderr]
return "".join(lines)
class ProcessTimedOut(Exception):
"""Raises by :func:`run_proc <plumbum.commands.run_proc>` when a ``timeout`` has been
specified and it has elapsed before the process terminated"""
def __init__(self, msg, argv):
Exception.__init__(self, msg, argv)
self.argv = argv
class ProcessLineTimedOut(Exception):
"""Raises by :func:`iter_lines <plumbum.commands.iter_lines>` when a ``line_timeout`` has been
specified and it has elapsed before the process yielded another line"""
def __init__(self, msg, argv, machine):
Exception.__init__(self, msg, argv, machine)
self.argv = argv
self.machine = machine
class CommandNotFound(AttributeError):
"""Raised by :func:`local.which <plumbum.machines.local.LocalMachine.which>` and
:func:`RemoteMachine.which <plumbum.machines.remote.RemoteMachine.which>` when a
command was not found in the system's ``PATH``"""
def __init__(self, program, path):
super().__init__(self, program, path)
self.program = program
self.path = path
# ===================================================================================================
# Timeout thread
# ===================================================================================================
class MinHeap:
def __init__(self, items=()):
self._items = list(items)
heapq.heapify(self._items)
def __len__(self):
return len(self._items)
def push(self, item):
heapq.heappush(self._items, item)
def pop(self):
heapq.heappop(self._items)
def peek(self):
return self._items[0]
_timeout_queue = Queue() # type: ignore[var-annotated]
_shutting_down = False
def _timeout_thread_func():
waiting = MinHeap()
try:
while not _shutting_down:
if waiting:
ttk, _ = waiting.peek()
timeout = max(0, ttk - time.time())
else:
timeout = None
with contextlib.suppress(QueueEmpty):
proc, time_to_kill = _timeout_queue.get(timeout=timeout)
if proc is SystemExit:
# terminate
return
waiting.push((time_to_kill, proc))
now = time.time()
while waiting:
ttk, proc = waiting.peek()
if ttk > now:
break
waiting.pop()
with contextlib.suppress(OSError):
if proc.poll() is None:
proc.kill()
proc._timed_out = True
except Exception:
if _shutting_down:
# to prevent all sorts of exceptions during interpreter shutdown
pass
else:
raise
bgthd = Thread(target=_timeout_thread_func, name="PlumbumTimeoutThread")
bgthd.daemon = True
bgthd.start()
def _register_proc_timeout(proc, timeout):
if timeout is not None:
_timeout_queue.put((proc, time.time() + timeout))
def _shutdown_bg_threads():
global _shutting_down # pylint: disable=global-statement
_shutting_down = True
# Make sure this still exists (don't throw error in atexit!)
# TODO: not sure why this would be "falsey", though
if _timeout_queue: # type: ignore[truthy-bool]
_timeout_queue.put((SystemExit, 0))
# grace period
bgthd.join(0.1)
atexit.register(_shutdown_bg_threads)
# ===================================================================================================
# run_proc
# ===================================================================================================
def run_proc(proc, retcode, timeout=None):
"""Waits for the given process to terminate, with the expected exit code
:param proc: a running Popen-like object, with all the expected methods.
:param retcode: the expected return (exit) code of the process. It defaults to 0 (the
convention for success). If ``None``, the return code is ignored.
It may also be a tuple (or any object that supports ``__contains__``)
of expected return codes.
:param timeout: the number of seconds (a ``float``) to allow the process to run, before
forcefully terminating it. If ``None``, not timeout is imposed; otherwise
the process is expected to terminate within that timeout value, or it will
be killed and :class:`ProcessTimedOut <plumbum.cli.ProcessTimedOut>`
will be raised
:returns: A tuple of (return code, stdout, stderr)
"""
_register_proc_timeout(proc, timeout)
stdout, stderr = proc.communicate()
proc._end_time = time.time()
if not stdout:
stdout = b""
if not stderr:
stderr = b""
if getattr(proc, "custom_encoding", None):
stdout = stdout.decode(proc.custom_encoding, "ignore")
stderr = stderr.decode(proc.custom_encoding, "ignore")
return _check_process(proc, retcode, timeout, stdout, stderr)
# ===================================================================================================
# iter_lines
# ===================================================================================================
BY_POSITION = object()
BY_TYPE = object()
DEFAULT_ITER_LINES_MODE = BY_POSITION
DEFAULT_BUFFER_SIZE = math.inf
def iter_lines(
proc,
retcode=0,
timeout=None,
linesize=-1,
line_timeout=None,
buffer_size=None,
mode=None,
_iter_lines=_iter_lines,
):
"""Runs the given process (equivalent to run_proc()) and yields a tuples of (out, err) line pairs.
If the exit code of the process does not match the expected one, :class:`ProcessExecutionError
<plumbum.commands.ProcessExecutionError>` is raised.
:param retcode: The expected return code of this process (defaults to 0).
In order to disable exit-code validation, pass ``None``. It may also
be a tuple (or any iterable) of expected exit codes.
:param timeout: The maximal amount of time (in seconds) to allow the process to run.
``None`` means no timeout is imposed; otherwise, if the process hasn't
terminated after that many seconds, the process will be forcefully
terminated an exception will be raised
:param linesize: Maximum number of characters to read from stdout/stderr at each iteration.
``-1`` (default) reads until a b'\\n' is encountered.
:param line_timeout: The maximal amount of time (in seconds) to allow between consecutive lines in either stream.
Raise an :class:`ProcessLineTimedOut <plumbum.commands.ProcessLineTimedOut>` if the timeout has
been reached. ``None`` means no timeout is imposed.
:param buffer_size: Maximum number of lines to keep in the stdout/stderr buffers, in case of a ProcessExecutionError.
Default is ``None``, which defaults to DEFAULT_BUFFER_SIZE (which is infinite by default).
``0`` will disable bufferring completely.
:param mode: Controls what the generator yields. Defaults to DEFAULT_ITER_LINES_MODE (which is BY_POSITION by default)
- BY_POSITION (default): yields ``(out, err)`` line tuples, where either item may be ``None``
- BY_TYPE: yields ``(fd, line)`` tuples, where ``fd`` is 1 (stdout) or 2 (stderr)
:returns: An iterator of (out, err) line tuples.
"""
if mode is None:
mode = DEFAULT_ITER_LINES_MODE
if buffer_size is None:
buffer_size = DEFAULT_BUFFER_SIZE
buffer_size: int
assert mode in (BY_POSITION, BY_TYPE)
encoding = getattr(proc, "custom_encoding", None) or "utf-8"
decode = lambda s: s.decode(encoding, errors="replace").rstrip() # noqa: E731
_register_proc_timeout(proc, timeout)
buffers = [[], []]
for t, line in _iter_lines(proc, decode, linesize, line_timeout):
# verify that the proc hasn't timed out yet
proc.verify(timeout=timeout, retcode=None, stdout=None, stderr=None)
buffer = buffers[t]
if buffer_size > 0:
buffer.append(line)
if buffer_size < math.inf:
del buffer[:-buffer_size]
if mode is BY_POSITION:
ret = [None, None]
ret[t] = line
yield tuple(ret)
elif mode is BY_TYPE:
yield (t + 1), line # 1=stdout, 2=stderr
# this will take care of checking return code and timeouts
_check_process(proc, retcode, timeout, *("\n".join(s) + "\n" for s in buffers))

View File

@@ -0,0 +1,3 @@
"""
file-system related operations
"""

View File

@@ -0,0 +1,318 @@
"""
Atomic file operations
"""
import atexit
import contextlib
import os
import threading
from plumbum.machines.local import local
try:
import fcntl
except ImportError:
import msvcrt
try:
from pywintypes import error as WinError
from win32con import LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY
from win32file import OVERLAPPED, LockFileEx, UnlockFile
except ImportError:
print( # noqa: T201
"On Windows, Plumbum requires Python for Windows Extensions (pywin32)"
)
raise
@contextlib.contextmanager
def locked_file(fileno, blocking=True):
hndl = msvcrt.get_osfhandle(fileno)
try:
LockFileEx(
hndl,
LOCKFILE_EXCLUSIVE_LOCK
| (0 if blocking else LOCKFILE_FAIL_IMMEDIATELY),
0xFFFFFFFF,
0xFFFFFFFF,
OVERLAPPED(),
)
except WinError as ex:
raise OSError(*ex.args) from None
try:
yield
finally:
UnlockFile(hndl, 0, 0, 0xFFFFFFFF, 0xFFFFFFFF)
else:
if hasattr(fcntl, "lockf"):
@contextlib.contextmanager
def locked_file(fileno, blocking=True):
fcntl.lockf(fileno, fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB))
try:
yield
finally:
fcntl.lockf(fileno, fcntl.LOCK_UN)
else:
@contextlib.contextmanager
def locked_file(fileno, blocking=True):
fcntl.flock(fileno, fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB))
try:
yield
finally:
fcntl.flock(fileno, fcntl.LOCK_UN)
class AtomicFile:
"""
Atomic file operations implemented using file-system advisory locks (``flock`` on POSIX,
``LockFile`` on Windows).
.. note::
On Linux, the manpage says ``flock`` might have issues with NFS mounts. You should
take this into account.
.. versionadded:: 1.3
"""
CHUNK_SIZE = 32 * 1024
def __init__(self, filename, ignore_deletion=False):
self.path = local.path(filename)
self._ignore_deletion = ignore_deletion
self._thdlock = threading.Lock()
self._owned_by = None
self._fileobj = None
self.reopen()
def __repr__(self):
return f"<AtomicFile: {self.path}>" if self._fileobj else "<AtomicFile: closed>"
def __del__(self):
self.close()
def __enter__(self):
return self
def __exit__(self, t, v, tb):
self.close()
def close(self):
if self._fileobj is not None:
self._fileobj.close()
self._fileobj = None
def reopen(self):
"""
Close and reopen the file; useful when the file was deleted from the file system
by a different process
"""
self.close()
self._fileobj = os.fdopen(
os.open(str(self.path), os.O_CREAT | os.O_RDWR, 384), "r+b", 0
)
@contextlib.contextmanager
def locked(self, blocking=True):
"""
A context manager that locks the file; this function is reentrant by the thread currently
holding the lock.
:param blocking: if ``True``, the call will block until we can grab the file system lock.
if ``False``, the call may fail immediately with the underlying exception
(``IOError`` or ``WindowsError``)
"""
if self._owned_by == threading.get_ident():
yield
return
with self._thdlock:
with locked_file(self._fileobj.fileno(), blocking):
if not self.path.exists() and not self._ignore_deletion:
raise ValueError("Atomic file removed from filesystem")
self._owned_by = threading.get_ident()
try:
yield
finally:
self._owned_by = None
def delete(self):
"""
Atomically delete the file (holds the lock while doing it)
"""
with self.locked():
self.path.delete()
def _read_all(self):
self._fileobj.seek(0)
data = []
while True:
buf = self._fileobj.read(self.CHUNK_SIZE)
data.append(buf)
if len(buf) < self.CHUNK_SIZE:
break
return b"".join(data)
def read_atomic(self):
"""Atomically read the entire file"""
with self.locked():
return self._read_all()
def read_shared(self):
"""Read the file **without** holding the lock"""
return self._read_all()
def write_atomic(self, data):
"""Writes the given data atomically to the file. Note that it overwrites the entire file;
``write_atomic("foo")`` followed by ``write_atomic("bar")`` will result in only ``"bar"``.
"""
with self.locked():
self._fileobj.seek(0)
while data:
chunk = data[: self.CHUNK_SIZE]
self._fileobj.write(chunk)
data = data[len(chunk) :]
self._fileobj.flush()
self._fileobj.truncate()
class AtomicCounterFile:
"""
An atomic counter based on AtomicFile. Each time you call ``next()``, it will
atomically read and increment the counter's value, returning its previous value
Example::
acf = AtomicCounterFile.open("/some/file")
print(acf.next()) # e.g., 7
print(acf.next()) # 8
print(acf.next()) # 9
.. versionadded:: 1.3
"""
def __init__(self, atomicfile, initial=0):
"""
:param atomicfile: an :class:`AtomicFile <plumbum.atomic.AtomicFile>` instance
:param initial: the initial value (used when the first time the file is created)
"""
self.atomicfile = atomicfile
self.initial = initial
def __enter__(self):
return self
def __exit__(self, t, v, tb):
self.close()
def close(self):
self.atomicfile.close()
@classmethod
def open(cls, filename):
"""
Shortcut for ``AtomicCounterFile(AtomicFile(filename))``
"""
return cls(AtomicFile(filename))
def reset(self, value=None):
"""
Reset the counter's value to the one given. If ``None``, it will default to the
initial value provided to the constructor
"""
if value is None:
value = self.initial
if not isinstance(value, int):
raise TypeError(f"value must be an integer, not {type(value)!r}")
self.atomicfile.write_atomic(str(value).encode("utf8"))
def next(self):
"""
Read and increment the counter, returning its previous value
"""
with self.atomicfile.locked():
curr = self.atomicfile.read_atomic().decode("utf8")
if not curr:
curr = self.initial
else:
curr = int(curr)
self.atomicfile.write_atomic(str(curr + 1).encode("utf8"))
return curr
class PidFileTaken(SystemExit):
"""
This exception is raised when PidFile.acquire fails to lock the pid file. Note that it
derives from ``SystemExit``, so unless explicitly handled, it will terminate the process
cleanly
"""
def __init__(self, msg, pid):
SystemExit.__init__(self, msg)
self.pid = pid
class PidFile:
"""
A PID file is a file that's locked by some process from the moment it starts until it dies
(the OS will clear the lock when the process exits). It is used to prevent two instances
of the same process (normally a daemon) from running concurrently. The PID file holds its
process' PID, so you know who's holding it.
.. versionadded:: 1.3
"""
def __init__(self, filename):
self.atomicfile = AtomicFile(filename)
self._ctx = None
def __enter__(self):
self.acquire()
def __exit__(self, t, v, tb):
self.release()
def __del__(self):
with contextlib.suppress(Exception):
self.release()
def close(self):
self.atomicfile.close()
def acquire(self):
"""
Attempt to acquire the PID file. If it's already locked, raises
:class:`PidFileTaken <plumbum.atomic.PidFileTaken>`. You should normally acquire
the file as early as possible when the program starts
"""
if self._ctx is not None:
return
self._ctx = self.atomicfile.locked(blocking=False)
try:
self._ctx.__enter__() # pylint: disable=unnecessary-dunder-call
except OSError:
self._ctx = None
try:
pid = self.atomicfile.read_shared().strip().decode("utf8")
except OSError:
pid = "Unknown"
raise PidFileTaken(
f"PID file {self.atomicfile.path!r} taken by process {pid}",
pid,
) from None
else:
self.atomicfile.write_atomic(str(os.getpid()).encode("utf8"))
atexit.register(self.release)
def release(self):
"""
Release the PID file (should only happen when the program terminates)
"""
if self._ctx is None:
return
self.atomicfile.delete()
try:
self._ctx.__exit__(None, None, None)
finally:
self._ctx = None

View File

@@ -0,0 +1,41 @@
import re
class MountEntry:
"""
Represents a mount entry (device file, mount point and file system type)
"""
def __init__(self, dev, point, fstype, options):
self.dev = dev
self.point = point
self.fstype = fstype
self.options = options.split(",")
def __str__(self):
options = ",".join(self.options)
return f"{self.dev} on {self.point} type {self.fstype} ({options})"
MOUNT_PATTERN = re.compile(r"(.+?)\s+on\s+(.+?)\s+type\s+(\S+)(?:\s+\((.+?)\))?")
def mount_table():
"""returns the system's current mount table (a list of
:class:`MountEntry <plumbum.unixutils.MountEntry>` objects)"""
from plumbum.cmd import mount
table = []
for line in mount().splitlines():
m = MOUNT_PATTERN.match(line)
if not m:
continue
table.append(MountEntry(*m.groups()))
return table
def mounted(fs):
"""
Indicates if a the given filesystem (device file or mount point) is currently mounted
"""
return any(fs in {entry.dev, entry.point} for entry in mount_table())

View File

@@ -0,0 +1,79 @@
import inspect
import os
import sys
from contextlib import contextmanager
from io import StringIO
IS_WIN32 = sys.platform == "win32"
class ProcInfo:
def __init__(self, pid, uid, stat, args):
self.pid = pid
self.uid = uid
self.stat = stat
self.args = args
def __repr__(self):
return f"ProcInfo({self.pid!r}, {self.uid!r}, {self.stat!r}, {self.args!r})"
@contextmanager
def captured_stdout(stdin=""):
"""
Captures stdout (similar to the redirect_stdout in Python 3.4+, but with slightly different arguments)
"""
prevstdin = sys.stdin
prevstdout = sys.stdout
sys.stdin = StringIO(stdin)
sys.stdout = StringIO()
try:
yield sys.stdout
finally:
sys.stdin = prevstdin
sys.stdout = prevstdout
class StaticProperty:
"""This acts like a static property, allowing access via class or object.
This is a non-data descriptor."""
def __init__(self, function):
self._function = function
self.__doc__ = function.__doc__
def __get__(self, obj, klass=None):
return self._function()
def getdoc(obj):
"""
This gets a docstring if available, and cleans it, but does not look up docs in
inheritance tree (Pre Python 3.5 behavior of ``inspect.getdoc``).
"""
try:
doc = obj.__doc__
except AttributeError:
return None
if not isinstance(doc, str):
return None
return inspect.cleandoc(doc)
def read_fd_decode_safely(fd, size=4096):
"""
This reads a utf-8 file descriptor and returns a chunk, growing up to
three bytes if needed to decode the character at the end.
Returns the data and the decoded text.
"""
data = os.read(fd.fileno(), size)
for _ in range(3):
try:
return data, data.decode("utf-8")
except UnicodeDecodeError as e:
if e.reason != "unexpected end of data":
raise
data += os.read(fd.fileno(), 1)
return data, data.decode("utf-8")

View File

@@ -0,0 +1,13 @@
from plumbum.machines.local import LocalCommand, LocalMachine, local
from plumbum.machines.remote import BaseRemoteMachine, RemoteCommand
from plumbum.machines.ssh_machine import PuttyMachine, SshMachine
__all__ = (
"LocalCommand",
"LocalMachine",
"local",
"BaseRemoteMachine",
"RemoteCommand",
"PuttyMachine",
"SshMachine",
)

View File

@@ -0,0 +1,26 @@
import struct
LFANEW_OFFSET = 30 * 2
FILE_HEADER_SIZE = 5 * 4
SUBSYSTEM_OFFSET = 17 * 4
IMAGE_SUBSYSTEM_WINDOWS_GUI = 2
IMAGE_SUBSYSTEM_WINDOWS_CUI = 3
def get_pe_subsystem(filename):
with open(filename, "rb") as f:
if f.read(2) != b"MZ":
return None
f.seek(LFANEW_OFFSET)
lfanew = struct.unpack("L", f.read(4))[0]
f.seek(lfanew)
if f.read(4) != b"PE\x00\x00":
return None
f.seek(FILE_HEADER_SIZE + SUBSYSTEM_OFFSET, 1)
subsystem = struct.unpack("H", f.read(2))[0]
return subsystem
# print(get_pe_subsystem("c:\\windows\\notepad.exe")) == 2
# print(get_pe_subsystem("c:\\python32\\python.exe")) == 3
# print(get_pe_subsystem("c:\\python32\\pythonw.exe")) == 2

View File

@@ -0,0 +1,102 @@
from plumbum.commands.processes import (
CommandNotFound,
ProcessExecutionError,
ProcessTimedOut,
)
class PopenAddons:
"""This adds a verify to popen objects to that the correct command is attributed when
an error is thrown."""
def verify(self, retcode, timeout, stdout, stderr):
"""This verifies that the correct command is attributed."""
if getattr(self, "_timed_out", False):
raise ProcessTimedOut(
f"Process did not terminate within {timeout} seconds",
getattr(self, "argv", None),
)
if retcode is not None:
if hasattr(retcode, "__contains__"):
if self.returncode not in retcode:
raise ProcessExecutionError(
getattr(self, "argv", None), self.returncode, stdout, stderr
)
elif self.returncode != retcode:
raise ProcessExecutionError(
getattr(self, "argv", None), self.returncode, stdout, stderr
)
class BaseMachine:
"""This is a base class for other machines. It contains common code to
all machines in Plumbum."""
def get(self, cmd, *othercommands):
"""This works a little like the ``.get`` method with dict's, only
it supports an unlimited number of arguments, since later arguments
are tried as commands and could also fail. It
will try to call the first command, and if that is not found,
it will call the next, etc. Will raise if no file named for the
executable if a path is given, unlike ``[]`` access.
Usage::
best_zip = local.get('pigz','gzip')
"""
try:
command = self[cmd]
if not command.executable.exists():
raise CommandNotFound(cmd, command.executable)
return command
except CommandNotFound:
if othercommands:
return self.get(othercommands[0], *othercommands[1:])
raise
def __contains__(self, cmd):
"""Tests for the existence of the command, e.g., ``"ls" in plumbum.local``.
``cmd`` can be anything acceptable by ``__getitem__``.
"""
try:
self[cmd]
except CommandNotFound:
return False
else:
return True
@property
def encoding(self):
"This is a wrapper for custom_encoding"
return self.custom_encoding
@encoding.setter
def encoding(self, value):
self.custom_encoding = value
def daemonic_popen(self, command, cwd="/", stdout=None, stderr=None, append=True):
raise NotImplementedError("This is not implemented on this machine!")
class Cmd:
def __init__(self, machine):
self._machine = machine
def __getattr__(self, name):
try:
return self._machine[name]
except CommandNotFound:
raise AttributeError(name) from None
@property
def cmd(self):
return self.Cmd(self)
def clear_program_cache(self):
"""
Clear the program cache, which is populated via ``machine.which(progname)`` calls.
This cache speeds up the lookup of a program in the machines PATH, and is particularly
effective for RemoteMachines.
"""
self._program_cache.clear()

View File

@@ -0,0 +1,187 @@
import os
from contextlib import contextmanager
class EnvPathList(list):
__slots__ = ["_path_factory", "_pathsep", "__weakref__"]
def __init__(self, path_factory, pathsep):
super().__init__()
self._path_factory = path_factory
self._pathsep = pathsep
def append(self, path):
list.append(self, self._path_factory(path))
def extend(self, paths):
list.extend(self, (self._path_factory(p) for p in paths))
def insert(self, index, path):
list.insert(self, index, self._path_factory(path))
def index(self, path):
list.index(self, self._path_factory(path))
def __contains__(self, path):
return list.__contains__(self, self._path_factory(path))
def remove(self, path):
list.remove(self, self._path_factory(path))
def update(self, text):
self[:] = [self._path_factory(p) for p in text.split(self._pathsep)]
def join(self):
return self._pathsep.join(str(p) for p in self)
class BaseEnv:
"""The base class of LocalEnv and RemoteEnv"""
__slots__ = ["_curr", "_path", "_path_factory", "__weakref__"]
CASE_SENSITIVE = True
def __init__(self, path_factory, pathsep, *, _curr):
self._curr = _curr
self._path_factory = path_factory
self._path = EnvPathList(path_factory, pathsep)
self._update_path()
def _update_path(self):
self._path.update(self.get("PATH", ""))
@contextmanager
def __call__(self, *args, **kwargs):
"""A context manager that can be used for temporal modifications of the environment.
Any time you enter the context, a copy of the old environment is stored, and then restored,
when the context exits.
:param args: Any positional arguments for ``update()``
:param kwargs: Any keyword arguments for ``update()``
"""
prev = self._curr.copy()
self.update(**kwargs)
try:
yield
finally:
self._curr = prev
self._update_path()
def __iter__(self):
"""Returns an iterator over the items ``(key, value)`` of current environment
(like dict.items)"""
return iter(self._curr.items())
def __hash__(self):
raise TypeError("unhashable type")
def __len__(self):
"""Returns the number of elements of the current environment"""
return len(self._curr)
def __contains__(self, name):
"""Tests whether an environment variable exists in the current environment"""
return (name if self.CASE_SENSITIVE else name.upper()) in self._curr
def __getitem__(self, name):
"""Returns the value of the given environment variable from current environment,
raising a ``KeyError`` if it does not exist"""
return self._curr[name if self.CASE_SENSITIVE else name.upper()]
def keys(self):
"""Returns the keys of the current environment (like dict.keys)"""
return self._curr.keys()
def items(self):
"""Returns the items of the current environment (like dict.items)"""
return self._curr.items()
def values(self):
"""Returns the values of the current environment (like dict.values)"""
return self._curr.values()
def get(self, name, *default):
"""Returns the keys of the current environment (like dict.keys)"""
return self._curr.get((name if self.CASE_SENSITIVE else name.upper()), *default)
def __delitem__(self, name):
"""Deletes an environment variable from the current environment"""
name = name if self.CASE_SENSITIVE else name.upper()
del self._curr[name]
if name == "PATH":
self._update_path()
def __setitem__(self, name, value):
"""Sets/replaces an environment variable's value in the current environment"""
name = name if self.CASE_SENSITIVE else name.upper()
self._curr[name] = value
if name == "PATH":
self._update_path()
def pop(self, name, *default):
"""Pops an element from the current environment (like dict.pop)"""
name = name if self.CASE_SENSITIVE else name.upper()
res = self._curr.pop(name, *default)
if name == "PATH":
self._update_path()
return res
def clear(self):
"""Clears the current environment (like dict.clear)"""
self._curr.clear()
self._update_path()
def update(self, *args, **kwargs):
"""Updates the current environment (like dict.update)"""
self._curr.update(*args, **kwargs)
if not self.CASE_SENSITIVE:
for k, v in list(self._curr.items()):
self._curr[k.upper()] = v
self._update_path()
def getdict(self):
"""Returns the environment as a real dictionary"""
self._curr["PATH"] = self.path.join()
return {k: str(v) for k, v in self._curr.items()}
@property
def path(self):
"""The system's ``PATH`` (as an easy-to-manipulate list)"""
return self._path
def _get_home(self):
if "HOME" in self:
return self._path_factory(self["HOME"])
if "USERPROFILE" in self: # pragma: no cover
return self._path_factory(self["USERPROFILE"])
if "HOMEPATH" in self: # pragma: no cover
return self._path_factory(self.get("HOMEDRIVE", ""), self["HOMEPATH"])
return None
def _set_home(self, p):
if "HOME" in self:
self["HOME"] = str(p)
elif "USERPROFILE" in self: # pragma: no cover
self["USERPROFILE"] = str(p)
elif "HOMEPATH" in self: # pragma: no cover
self["HOMEPATH"] = str(p)
else: # pragma: no cover
self["HOME"] = str(p)
home = property(_get_home, _set_home)
"""Get or set the home path"""
@property
def user(self):
"""Return the user name, or ``None`` if it is not set"""
# adapted from getpass.getuser()
for name in ("LOGNAME", "USER", "LNAME", "USERNAME"): # pragma: no branch
if name in self:
return self[name]
try:
# POSIX only
import pwd
except ImportError:
return None
else:
return pwd.getpwuid(os.getuid())[0] # @UndefinedVariable

View File

@@ -0,0 +1,461 @@
import contextlib
import logging
import os
import platform
import re
import subprocess
import sys
import time
from contextlib import contextmanager
from subprocess import PIPE, Popen
from tempfile import mkdtemp
from typing import Dict, Tuple
from plumbum.commands import CommandNotFound, ConcreteCommand
from plumbum.commands.daemons import posix_daemonize, win32_daemonize
from plumbum.commands.processes import iter_lines
from plumbum.lib import IS_WIN32, ProcInfo, StaticProperty
from plumbum.machines.base import BaseMachine, PopenAddons
from plumbum.machines.env import BaseEnv
from plumbum.machines.session import ShellSession
from plumbum.path.local import LocalPath, LocalWorkdir
from plumbum.path.remote import RemotePath
class PlumbumLocalPopen(PopenAddons):
iter_lines = iter_lines
def __init__(self, *args, **kwargs):
self._proc = Popen(*args, **kwargs) # pylint: disable=consider-using-with
def __iter__(self):
return self.iter_lines()
def __enter__(self):
return self._proc.__enter__()
def __exit__(self, *args, **kwargs):
return self._proc.__exit__(*args, **kwargs)
def __getattr__(self, name):
return getattr(self._proc, name)
if IS_WIN32:
from plumbum.machines._windows import IMAGE_SUBSYSTEM_WINDOWS_CUI, get_pe_subsystem
logger = logging.getLogger("plumbum.local")
# ===================================================================================================
# Environment
# ===================================================================================================
class LocalEnv(BaseEnv):
"""The local machine's environment; exposes a dict-like interface"""
__slots__ = ()
CASE_SENSITIVE = not IS_WIN32
def __init__(self):
# os.environ already takes care of upper'ing on windows
super().__init__(LocalPath, os.path.pathsep, _curr=os.environ.copy())
if IS_WIN32 and "HOME" not in self and self.home is not None:
self["HOME"] = self.home
def expand(self, expr):
"""Expands any environment variables and home shortcuts found in ``expr``
(like ``os.path.expanduser`` combined with ``os.path.expandvars``)
:param expr: An expression containing environment variables (as ``$FOO``) or
home shortcuts (as ``~/.bashrc``)
:returns: The expanded string"""
prev = os.environ
os.environ = self.getdict() # noqa: B003
try:
output = os.path.expanduser(os.path.expandvars(expr))
finally:
os.environ = prev # noqa: B003
return output
def expanduser(self, expr):
"""Expand home shortcuts (e.g., ``~/foo/bar`` or ``~john/foo/bar``)
:param expr: An expression containing home shortcuts
:returns: The expanded string"""
prev = os.environ
os.environ = self.getdict() # noqa: B003
try:
output = os.path.expanduser(expr)
finally:
os.environ = prev # noqa: B003
return output
# ===================================================================================================
# Local Commands
# ===================================================================================================
class LocalCommand(ConcreteCommand):
__slots__ = ()
QUOTE_LEVEL = 2
def __init__(self, executable, encoding="auto"):
ConcreteCommand.__init__(
self, executable, local.custom_encoding if encoding == "auto" else encoding
)
@property
def machine(self):
return local
def popen(self, args=(), cwd=None, env=None, **kwargs):
if isinstance(args, str):
args = (args,)
return self.machine._popen(
self.executable,
self.formulate(0, args),
cwd=self.cwd if cwd is None else cwd,
env=self.env if env is None else env,
**kwargs,
)
# ===================================================================================================
# Local Machine
# ===================================================================================================
class LocalMachine(BaseMachine):
"""The *local machine* (a singleton object). It serves as an entry point to everything
related to the local machine, such as working directory and environment manipulation,
command creation, etc.
Attributes:
* ``cwd`` - the local working directory
* ``env`` - the local environment
* ``custom_encoding`` - the local machine's default encoding (``sys.getfilesystemencoding()``)
"""
cwd = StaticProperty(LocalWorkdir)
env = LocalEnv()
custom_encoding = sys.getfilesystemencoding()
uname = platform.uname()[0]
_program_cache: Dict[Tuple[str, str], LocalPath] = {}
def __init__(self):
self._as_user_stack = []
if IS_WIN32:
_EXTENSIONS = [""] + env.get("PATHEXT", ":.exe:.bat").lower().split(
os.path.pathsep
)
@classmethod
def _which(cls, progname):
progname = progname.lower()
for p in cls.env.path:
for ext in cls._EXTENSIONS:
fn = p / (progname + ext)
if fn.access("x") and not fn.is_dir():
return fn
return None
else:
@classmethod
def _which(cls, progname):
for p in cls.env.path:
fn = p / progname
if fn.access("x") and not fn.is_dir():
return fn
return None
@classmethod
def which(cls, progname):
"""Looks up a program in the ``PATH``. If the program is not found, raises
:class:`CommandNotFound <plumbum.commands.CommandNotFound>`
:param progname: The program's name. Note that if underscores (``_``) are present
in the name, and the exact name is not found, they will be replaced
in turn by hyphens (``-``) then periods (``.``), and the name will
be looked up again for each alternative
:returns: A :class:`LocalPath <plumbum.machines.local.LocalPath>`
"""
key = (progname, cls.env.get("PATH", ""))
with contextlib.suppress(KeyError):
return cls._program_cache[key]
alternatives = [progname]
if "_" in progname:
alternatives += [progname.replace("_", "-"), progname.replace("_", ".")]
for pn in alternatives:
path = cls._which(pn)
if path:
cls._program_cache[key] = path
return path
raise CommandNotFound(progname, list(cls.env.path))
def path(self, *parts):
"""A factory for :class:`LocalPaths <plumbum.path.local.LocalPath>`.
Usage: ``p = local.path("/usr", "lib", "python2.7")``
"""
parts2 = [str(self.cwd)]
for p in parts:
if isinstance(p, RemotePath):
raise TypeError(f"Cannot construct LocalPath from {p!r}")
parts2.append(self.env.expanduser(str(p)))
return LocalPath(os.path.join(*parts2))
def __contains__(self, cmd):
try:
self[cmd]
except CommandNotFound:
return False
else:
return True
def __getitem__(self, cmd):
"""Returns a `Command` object representing the given program. ``cmd`` can be a string or
a :class:`LocalPath <plumbum.path.local.LocalPath>`; if it is a path, a command
representing this path will be returned; otherwise, the program name will be looked up
in the system's ``PATH`` (using ``which``). Usage::
ls = local["ls"]
"""
if isinstance(cmd, LocalPath):
return LocalCommand(cmd)
if not isinstance(cmd, RemotePath):
# handle "path-like" (pathlib.Path) objects
cmd = os.fspath(cmd)
if "/" in cmd or "\\" in cmd:
# assume path
return LocalCommand(local.path(cmd))
# search for command
return LocalCommand(self.which(cmd))
raise TypeError(f"cmd must not be a RemotePath: {cmd!r}")
def _popen(
self,
executable,
argv,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
cwd=None,
env=None,
new_session=False,
**kwargs,
):
if new_session:
kwargs["start_new_session"] = True
if IS_WIN32 and "startupinfo" not in kwargs and stdin not in (sys.stdin, None):
subsystem = get_pe_subsystem(str(executable))
if subsystem == IMAGE_SUBSYSTEM_WINDOWS_CUI:
# don't open a new console
sui = subprocess.STARTUPINFO()
kwargs["startupinfo"] = sui
if hasattr(subprocess, "_subprocess"):
sui.dwFlags |= (
subprocess._subprocess.STARTF_USESHOWWINDOW
) # @UndefinedVariable
sui.wShowWindow = (
subprocess._subprocess.SW_HIDE
) # @UndefinedVariable
else:
sui.dwFlags |= subprocess.STARTF_USESHOWWINDOW # @UndefinedVariable
sui.wShowWindow = subprocess.SW_HIDE # @UndefinedVariable
if cwd is None:
cwd = self.cwd
envs = [self.env, env]
env = {}
for _env in envs:
if not _env:
continue
if isinstance(_env, BaseEnv):
_env = _env.getdict()
env.update(_env)
if self._as_user_stack:
argv, executable = self._as_user_stack[-1](argv)
logger.debug("Running %r", argv)
proc = PlumbumLocalPopen(
argv,
executable=str(executable),
stdin=stdin,
stdout=stdout,
stderr=stderr,
cwd=str(cwd),
env=env,
**kwargs,
) # bufsize = 4096
proc._start_time = time.time()
proc.custom_encoding = self.custom_encoding
proc.argv = argv
return proc
def daemonic_popen(self, command, cwd="/", stdout=None, stderr=None, append=True):
"""
On POSIX systems:
Run ``command`` as a UNIX daemon: fork a child process to setpid, redirect std handles to /dev/null,
umask, close all fds, chdir to ``cwd``, then fork and exec ``command``. Returns a ``Popen`` process that
can be used to poll/wait for the executed command (but keep in mind that you cannot access std handles)
On Windows:
Run ``command`` as a "Windows daemon": detach from controlling console and create a new process group.
This means that the command will not receive console events and would survive its parent's termination.
Returns a ``Popen`` object.
.. note:: this does not run ``command`` as a system service, only detaches it from its parent.
.. versionadded:: 1.3
"""
if IS_WIN32:
return win32_daemonize(command, cwd, stdout, stderr, append)
return posix_daemonize(command, cwd, stdout, stderr, append)
if IS_WIN32:
def list_processes(self): # pylint: disable=no-self-use
"""
Returns information about all running processes (on Windows: using ``tasklist``)
.. versionadded:: 1.3
"""
import csv
tasklist = local["tasklist"]
output = tasklist("/V", "/FO", "CSV")
lines = output.splitlines()
rows = csv.reader(lines)
try:
header = next(rows)
except StopIteration:
raise RuntimeError("tasklist must at least have header") from None
imgidx = header.index("Image Name")
pididx = header.index("PID")
statidx = header.index("Status")
useridx = header.index("User Name")
for row in rows:
yield ProcInfo(
int(row[pididx]), row[useridx], row[statidx], row[imgidx]
)
else:
def list_processes(self):
"""
Returns information about all running processes (on POSIX systems: using ``ps``)
.. versionadded:: 1.3
"""
ps = self["ps"]
lines = ps("-e", "-o", "pid,uid,stat,args").splitlines()
lines.pop(0) # header
for line in lines:
parts = line.strip().split()
yield ProcInfo(
int(parts[0]), int(parts[1]), parts[2], " ".join(parts[3:])
)
def pgrep(self, pattern):
"""
Process grep: return information about all processes whose command-line args match the given regex pattern
"""
pat = re.compile(pattern)
for procinfo in self.list_processes():
if pat.search(procinfo.args):
yield procinfo
def session(self, new_session=False):
"""Creates a new :class:`ShellSession <plumbum.session.ShellSession>` object; this
invokes ``/bin/sh`` and executes commands on it over stdin/stdout/stderr"""
return ShellSession(self["sh"].popen(new_session=new_session))
@contextmanager
def tempdir(self):
"""A context manager that creates a temporary directory, which is removed when the context
exits"""
new_dir = self.path(mkdtemp())
try:
yield new_dir
finally:
new_dir.delete()
@contextmanager
def as_user(self, username=None):
"""Run nested commands as the given user. For example::
head = local["head"]
head("-n1", "/dev/sda1") # this will fail...
with local.as_user():
head("-n1", "/dev/sda1")
:param username: The user to run commands as. If not given, root (or Administrator) is assumed
"""
if IS_WIN32:
if username is None:
username = "Administrator"
self._as_user_stack.append(
lambda argv: (
[
"runas",
"/savecred",
f"/user:{username}",
'"' + " ".join(str(a) for a in argv) + '"',
],
self.which("runas"),
)
)
else:
if username is None:
self._as_user_stack.append(
lambda argv: (["sudo"] + list(argv), self.which("sudo"))
)
else:
self._as_user_stack.append(
lambda argv: (
["sudo", "-u", username] + list(argv),
self.which("sudo"),
)
)
try:
yield
finally:
self._as_user_stack.pop(-1)
def as_root(self):
"""A shorthand for :func:`as_user("root") <plumbum.machines.local.LocalMachine.as_user>`"""
return self.as_user()
python = LocalCommand(sys.executable, custom_encoding)
"""A command that represents the current python interpreter (``sys.executable``)"""
local = LocalMachine()
"""The *local machine* (a singleton object). It serves as an entry point to everything
related to the local machine, such as working directory and environment manipulation,
command creation, etc.
Attributes:
* ``cwd`` - the local working directory
* ``env`` - the local environment
* ``custom_encoding`` - the local machine's default encoding (``sys.getfilesystemencoding()``)
"""

View File

@@ -0,0 +1,515 @@
import contextlib
import errno
import logging
import os
import stat
from plumbum.commands.base import shquote
from plumbum.commands.processes import ProcessLineTimedOut, iter_lines
from plumbum.machines.base import PopenAddons
from plumbum.machines.remote import BaseRemoteMachine
from plumbum.machines.session import ShellSession
from plumbum.path.local import LocalPath
from plumbum.path.remote import RemotePath, StatRes
try:
# Sigh... we need to gracefully-import paramiko for Sphinx builds, etc
import paramiko
except ImportError:
class paramiko: # type: ignore[no-redef]
def __bool__(self):
return False
def __getattr__(self, name):
raise ImportError("No module named paramiko")
paramiko = paramiko() # type: ignore[operator]
logger = logging.getLogger("plumbum.paramiko")
class ParamikoPopen(PopenAddons):
def __init__(
self,
argv,
stdin,
stdout,
stderr,
encoding,
stdin_file=None,
stdout_file=None,
stderr_file=None,
):
self.argv = argv
self.channel = stdout.channel
self.stdin = stdin
self.stdout = stdout
self.stderr = stderr
self.custom_encoding = encoding
self.returncode = None
self.pid = None
self.stdin_file = stdin_file
self.stdout_file = stdout_file
self.stderr_file = stderr_file
def poll(self):
if self.returncode is None:
if self.channel.exit_status_ready():
return self.wait()
return self.returncode
def wait(self):
if self.returncode is None:
self.channel.recv_exit_status()
self.returncode = self.channel.exit_status
self.close()
return self.returncode
def close(self):
self.channel.shutdown_read()
self.channel.shutdown_write()
self.channel.close()
@staticmethod
def kill():
# possible way to obtain pid:
# "(cmd ; echo $?) & echo ?!"
# and then client.exec_command("kill -9 %s" % (pid,))
raise OSError("Cannot kill remote processes, we don't have their PIDs")
terminate = kill
def send_signal(self, sig):
raise NotImplementedError()
def communicate(self):
stdout = []
stderr = []
infile = self.stdin_file
sources = [
("1", stdout, self.stdout, self.stdout_file),
("2", stderr, self.stderr, self.stderr_file),
]
i = 0
while sources:
if infile:
try:
line = infile.readline()
except (ValueError, OSError):
line = None
logger.debug("communicate: %r", line)
if not line:
infile.close()
infile = None
self.stdin.close()
else:
self.stdin.write(line)
self.stdin.flush()
i = (i + 1) % len(sources)
_name, coll, pipe, outfile = sources[i]
line = pipe.readline()
# logger.debug("%s> %r", name, line)
if not line:
del sources[i]
elif outfile:
outfile.write(line)
outfile.flush()
else:
coll.append(line)
self.wait()
stdout = "".join(s for s in stdout).encode(self.custom_encoding)
stderr = "".join(s for s in stderr).encode(self.custom_encoding)
return stdout, stderr
def iter_lines(self, timeout=None, **kwargs):
if timeout is not None:
raise NotImplementedError(
"The 'timeout' parameter is not supported with ParamikoMachine"
)
return iter_lines(self, _iter_lines=_iter_lines, **kwargs)
__iter__ = iter_lines
class ParamikoMachine(BaseRemoteMachine):
"""
An implementation of :class:`remote machine <plumbum.machines.remote.BaseRemoteMachine>`
over Paramiko (a Python implementation of openSSH2 client/server). Invoking a remote command
translates to invoking it over SSH ::
with ParamikoMachine("yourhostname") as rem:
r_ls = rem["ls"]
# r_ls is the remote `ls`
# executing r_ls() is equivalent to `ssh yourhostname ls`, only without
# spawning a new ssh client
:param host: the host name to connect to (SSH server)
:param user: the user to connect as (if ``None``, the default will be used)
:param port: the server's port (if ``None``, the default will be used)
:param password: the user's password (if a password-based authentication is to be performed)
(if ``None``, key-based authentication will be used)
:param keyfile: the path to the identity file (if ``None``, the default will be used)
:param load_system_host_keys: whether or not to load the system's host keys (from ``/etc/ssh``
and ``~/.ssh``). The default is ``True``, which means Paramiko
behaves much like the ``ssh`` command-line client
:param missing_host_policy: the value passed to the underlying ``set_missing_host_key_policy``
of the client. The default is ``None``, which means
``set_missing_host_key_policy`` is not invoked and paramiko's
default behavior (reject) is employed
:param encoding: the remote machine's encoding (defaults to UTF8)
:param look_for_keys: set to False to disable searching for discoverable
private key files in ``~/.ssh``
:param connect_timeout: timeout for TCP connection
.. note:: If Paramiko 1.15 or above is installed, can use GSS_API authentication
:param bool gss_auth: ``True`` if you want to use GSS-API authentication
:param bool gss_kex: Perform GSS-API Key Exchange and user authentication
:param bool gss_deleg_creds: Delegate GSS-API client credentials or not
:param str gss_host: The targets name in the kerberos database. default: hostname
:param bool get_pty: Execute remote commands with allocated pseudo-tty. default: False
:param bool load_system_ssh_config: read system SSH config for ProxyCommand configuration. default: False
"""
class RemoteCommand(BaseRemoteMachine.RemoteCommand): # type: ignore[valid-type, misc]
def __or__(self, *_):
raise NotImplementedError("Not supported with ParamikoMachine")
def __gt__(self, *_):
raise NotImplementedError("Not supported with ParamikoMachine")
def __rshift__(self, *_):
raise NotImplementedError("Not supported with ParamikoMachine")
def __ge__(self, *_):
raise NotImplementedError("Not supported with ParamikoMachine")
def __lt__(self, *_):
raise NotImplementedError("Not supported with ParamikoMachine")
def __lshift__(self, *_):
raise NotImplementedError("Not supported with ParamikoMachine")
def __init__(
self,
host,
user=None,
port=None,
password=None,
keyfile=None,
load_system_host_keys=True,
missing_host_policy=None,
encoding="utf8",
look_for_keys=None,
connect_timeout=None,
keep_alive=0,
gss_auth=False,
gss_kex=None,
gss_deleg_creds=None,
gss_host=None,
get_pty=False,
load_system_ssh_config=False,
):
self.host = host
kwargs = {}
if user:
self._fqhost = f"{user}@{host}"
kwargs["username"] = user
else:
self._fqhost = host
self._client = paramiko.SSHClient()
if load_system_host_keys:
self._client.load_system_host_keys()
if port is not None:
kwargs["port"] = port
if keyfile is not None:
kwargs["key_filename"] = keyfile
if password is not None:
kwargs["password"] = password
if missing_host_policy is not None:
self._client.set_missing_host_key_policy(missing_host_policy)
if look_for_keys is not None:
kwargs["look_for_keys"] = look_for_keys
if connect_timeout is not None:
kwargs["timeout"] = connect_timeout
if gss_auth:
kwargs["gss_auth"] = gss_auth
kwargs["gss_kex"] = gss_kex
kwargs["gss_deleg_creds"] = gss_deleg_creds
if not gss_host:
gss_host = host
kwargs["gss_host"] = gss_host
if load_system_ssh_config:
ssh_config = paramiko.SSHConfig()
with open(os.path.expanduser("~/.ssh/config"), encoding="utf-8") as f:
ssh_config.parse(f)
with contextlib.suppress(KeyError):
hostConfig = ssh_config.lookup(host)
kwargs["sock"] = paramiko.ProxyCommand(hostConfig["proxycommand"])
self._client.connect(host, **kwargs)
self._keep_alive = keep_alive
self._sftp = None
self._get_pty = get_pty
BaseRemoteMachine.__init__(self, encoding, connect_timeout)
def __str__(self):
return f"paramiko://{self._fqhost}"
def close(self):
BaseRemoteMachine.close(self)
self._client.close()
@property
def sftp(self):
"""
Returns an SFTP client on top of the current SSH connection; it can be used to manipulate
files directly, much like an interactive FTP/SFTP session
"""
if not self._sftp:
self._sftp = self._client.open_sftp()
return self._sftp
def session(
self, isatty=False, term="vt100", width=80, height=24, *, new_session=False
):
# new_session is ignored for ParamikoMachine
trans = self._client.get_transport()
trans.set_keepalive(self._keep_alive)
chan = trans.open_session()
if isatty:
chan.get_pty(term, width, height)
chan.set_combine_stderr(True)
chan.invoke_shell()
stdin = chan.makefile("wb", -1)
stdout = chan.makefile("rb", -1)
stderr = chan.makefile_stderr("rb", -1)
proc = ParamikoPopen(["<shell>"], stdin, stdout, stderr, self.custom_encoding)
return ShellSession(proc, self.custom_encoding, isatty)
def popen(
self,
args,
stdin=None,
stdout=None,
stderr=None,
new_session=False, # pylint: disable=unused-argument
env=None,
cwd=None,
):
# new_session is ignored for ParamikoMachine
argv = []
envdelta = self.env.getdelta()
if env:
envdelta.update(env)
argv.extend(["cd", str(cwd or self.cwd), "&&"])
if envdelta:
argv.append("env")
argv.extend(f"{k}={shquote(v)}" for k, v in envdelta.items())
argv.extend(args.formulate())
cmdline = " ".join(argv)
logger.debug(cmdline)
si, so, se = self._client.exec_command(cmdline, 1, get_pty=self._get_pty)
return ParamikoPopen(
argv,
si,
so,
se,
self.custom_encoding,
stdin_file=stdin,
stdout_file=stdout,
stderr_file=stderr,
)
def download(self, src, dst):
if isinstance(src, LocalPath):
raise TypeError(f"src of download cannot be {src!r}")
if isinstance(src, RemotePath) and src.remote != self:
raise TypeError(f"src {src!r} points to a different remote machine")
if isinstance(dst, RemotePath):
raise TypeError(f"dst of download cannot be {dst!r}")
return self._download(
src if isinstance(src, RemotePath) else self.path(src),
dst if isinstance(dst, LocalPath) else LocalPath(dst),
)
def _download(self, src, dst):
if src.is_dir():
if not dst.exists():
self.sftp.mkdir(str(dst))
for fn in src:
self._download(fn, dst / fn.name)
elif dst.is_dir():
self.sftp.get(str(src), str(dst / src.name))
else:
self.sftp.get(str(src), str(dst))
def upload(self, src, dst):
if isinstance(src, RemotePath):
raise TypeError(f"src of upload cannot be {src!r}")
if isinstance(dst, LocalPath):
raise TypeError(f"dst of upload cannot be {dst!r}")
if isinstance(dst, RemotePath) and dst.remote != self:
raise TypeError(f"dst {dst!r} points to a different remote machine")
return self._upload(
src if isinstance(src, LocalPath) else LocalPath(src),
dst if isinstance(dst, RemotePath) else self.path(dst),
)
def _upload(self, src, dst):
if src.is_dir():
if not dst.exists():
self.sftp.mkdir(str(dst))
for fn in src:
self._upload(fn, dst / fn.name)
elif dst.is_dir():
self.sftp.put(str(src), str(dst / src.name))
else:
self.sftp.put(str(src), str(dst))
def connect_sock(self, dport, dhost="localhost", ipv6=False):
"""Returns a Paramiko ``Channel``, connected to dhost:dport on the remote machine.
The ``Channel`` behaves like a regular socket; you can ``send`` and ``recv`` on it
and the data will pass encrypted over SSH. Usage::
mach = ParamikoMachine("myhost")
sock = mach.connect_sock(12345)
data = sock.recv(100)
sock.send("foobar")
sock.close()
"""
if ipv6 and dhost == "localhost":
dhost = "::1"
srcaddr = ("::1", 0, 0, 0) if ipv6 else ("127.0.0.1", 0)
trans = self._client.get_transport()
trans.set_keepalive(self._keep_alive)
chan = trans.open_channel("direct-tcpip", (dhost, dport), srcaddr)
return SocketCompatibleChannel(chan)
#
# Path implementation
#
def _path_listdir(self, fn):
return self.sftp.listdir(str(fn))
def _path_read(self, fn):
f = self.sftp.open(str(fn), "rb")
data = f.read()
f.close()
return data
def _path_write(self, fn, data):
if self.custom_encoding and isinstance(data, str):
data = data.encode(self.custom_encoding)
f = self.sftp.open(str(fn), "wb")
f.write(data)
f.close()
def _path_stat(self, fn):
try:
st = self.sftp.stat(str(fn))
except OSError as e:
if e.errno == errno.ENOENT:
return None
raise
res = StatRes(
(
st.st_mode,
0,
0,
0,
st.st_uid,
st.st_gid,
st.st_size,
st.st_atime,
st.st_mtime,
0,
)
)
if stat.S_ISDIR(st.st_mode):
res.text_mode = "directory"
if stat.S_ISREG(st.st_mode):
res.text_mode = "regular file"
return res
def daemonic_popen(self, command, cwd="/", stdout=None, stderr=None, append=True):
raise NotImplementedError("This is not implemented on ParamikoMachine!")
###################################################################################################
# Make paramiko.Channel adhere to the socket protocol, namely, send and recv should fail
# when the socket has been closed
###################################################################################################
class SocketCompatibleChannel:
def __init__(self, chan):
self._chan = chan
def __getattr__(self, name):
return getattr(self._chan, name)
def send(self, s):
if self._chan.closed:
raise OSError(errno.EBADF, "Bad file descriptor")
return self._chan.send(s)
def recv(self, count):
if self._chan.closed:
raise OSError(errno.EBADF, "Bad file descriptor")
return self._chan.recv(count)
###################################################################################################
# Custom iter_lines for paramiko.Channel
###################################################################################################
def _iter_lines(
proc,
decode, # pylint: disable=unused-argument
linesize,
line_timeout=None,
):
from selectors import EVENT_READ, DefaultSelector
# Python 3.4+ implementation
def selector():
sel = DefaultSelector()
sel.register(proc.stdout.channel, EVENT_READ)
while True:
ready = sel.select(line_timeout)
if not ready and line_timeout:
raise ProcessLineTimedOut(
"popen line timeout expired",
getattr(proc, "argv", None),
getattr(proc, "machine", None),
)
for _key, _mask in ready:
yield
for _ in selector():
if proc.stdout.channel.recv_ready():
yield 0, proc.stdout.readline(linesize)
if proc.stdout.channel.recv_stderr_ready():
yield 1, proc.stderr.readline(linesize)
if proc.poll() is not None:
break
for line in proc.stdout:
yield 0, line
for line in proc.stderr:
yield 1, line

View File

@@ -0,0 +1,451 @@
import contextlib
import re
from tempfile import NamedTemporaryFile
from plumbum.commands import CommandNotFound, ConcreteCommand, shquote
from plumbum.lib import ProcInfo
from plumbum.machines.base import BaseMachine
from plumbum.machines.env import BaseEnv
from plumbum.path.local import LocalPath
from plumbum.path.remote import RemotePath, RemoteWorkdir, StatRes
class RemoteEnv(BaseEnv):
"""The remote machine's environment; exposes a dict-like interface"""
__slots__ = ["_orig", "remote"]
def __init__(self, remote):
session = remote._session
# GNU env has a -0 argument; use it if present. Otherwise,
# fall back to calling printenv on each (possible) variable
# from plain env.
env0 = session.run("env -0; echo")
if env0[0] == 0 and not env0[2].rstrip():
_curr = dict(
line.split("=", 1) for line in env0[1].split("\x00") if "=" in line
)
else:
lines = session.run("env; echo")[1].splitlines()
split = (line.split("=", 1) for line in lines)
keys = (line[0] for line in split if len(line) > 1)
runs = ((key, session.run(f'printenv "{key}"; echo')) for key in keys)
_curr = {
key: run[1].rstrip("\n")
for (key, run) in runs
if run[0] == 0 and run[1].rstrip("\n") and not run[2]
}
super().__init__(remote.path, ":", _curr=_curr)
self.remote = remote
self._orig = self._curr.copy()
def __delitem__(self, name):
BaseEnv.__delitem__(self, name)
self.remote._session.run(f"unset {name}")
def __setitem__(self, name, value):
BaseEnv.__setitem__(self, name, value)
self.remote._session.run(f"export {name}={shquote(value)}")
def pop(self, name, *default):
BaseEnv.pop(self, name, *default)
self.remote._session.run(f"unset {name}")
def update(self, *args, **kwargs):
BaseEnv.update(self, *args, **kwargs)
self.remote._session.run(
"export " + " ".join(f"{k}={shquote(v)}" for k, v in self.getdict().items())
)
def expand(self, expr):
"""Expands any environment variables and home shortcuts found in ``expr``
(like ``os.path.expanduser`` combined with ``os.path.expandvars``)
:param expr: An expression containing environment variables (as ``$FOO``) or
home shortcuts (as ``~/.bashrc``)
:returns: The expanded string"""
return self.remote.expand(expr)
def expanduser(self, expr):
"""Expand home shortcuts (e.g., ``~/foo/bar`` or ``~john/foo/bar``)
:param expr: An expression containing home shortcuts
:returns: The expanded string"""
return self.remote.expanduser(expr)
# def clear(self):
# BaseEnv.clear(self, *args, **kwargs)
# self.remote._session.run("export %s" % " ".join("%s=%s" % (k, v) for k, v in self.getdict()))
def getdelta(self):
"""Returns the difference between the this environment and the original environment of
the remote machine"""
self._curr["PATH"] = self.path.join()
delta = {}
for k, v in self._curr.items():
if k not in self._orig:
delta[k] = str(v)
for k, v in self._orig.items():
if k not in self._curr:
delta[k] = ""
else:
if v != self._curr[k]:
delta[k] = self._curr[k]
return delta
class RemoteCommand(ConcreteCommand):
__slots__ = ("remote",)
QUOTE_LEVEL = 1
def __init__(self, remote, executable, encoding="auto"):
self.remote = remote
ConcreteCommand.__init__(
self, executable, remote.custom_encoding if encoding == "auto" else encoding
)
@property
def machine(self):
return self.remote
def __repr__(self):
return f"RemoteCommand({self.remote!r}, {self.executable!r})"
def popen(self, args=(), **kwargs):
return self.remote.popen(self[args], **kwargs)
def nohup(self, cwd=".", stdout="nohup.out", stderr=None, append=True):
"""Runs a command detached."""
return self.machine.daemonic_popen(self, cwd, stdout, stderr, append)
class ClosedRemoteMachine(Exception):
pass
class ClosedRemote:
__slots__ = ["_obj", "__weakref__"]
def __init__(self, obj):
self._obj = obj
def close(self):
pass
def __getattr__(self, name):
raise ClosedRemoteMachine(f"{self._obj!r} has been closed")
class BaseRemoteMachine(BaseMachine):
"""Represents a *remote machine*; serves as an entry point to everything related to that
remote machine, such as working directory and environment manipulation, command creation,
etc.
Attributes:
* ``cwd`` - the remote working directory
* ``env`` - the remote environment
* ``custom_encoding`` - the remote machine's default encoding (assumed to be UTF8)
* ``connect_timeout`` - the connection timeout
There also is a _cwd attribute that exists if the cwd is not current (del if cwd is changed).
"""
# allow inheritors to override the RemoteCommand class
RemoteCommand = RemoteCommand
@property
def cwd(self):
if not hasattr(self, "_cwd"):
self._cwd = RemoteWorkdir(self)
return self._cwd
def __init__(self, encoding="utf8", connect_timeout=10, new_session=False):
self.custom_encoding = encoding
self.connect_timeout = connect_timeout
self._session = self.session(new_session=new_session)
self.uname = self._get_uname()
self.env = RemoteEnv(self)
self._python = None
self._program_cache = {}
def _get_uname(self):
rc, out, _ = self._session.run("uname", retcode=None)
if rc == 0:
return out.strip()
rc, out, _ = self._session.run(
"python3 -c 'import platform;print(platform.uname()[0])'", retcode=None
)
if rc == 0:
return out.strip()
# all POSIX systems should have uname. make an educated guess it's Windows
return "Windows"
def __repr__(self):
return f"<{self.__class__.__name__} {self}>"
def __enter__(self):
return self
def __exit__(self, t, v, tb):
self.close()
def close(self):
"""closes the connection to the remote machine; all paths and programs will
become defunct"""
self._session.close()
self._session = ClosedRemote(self)
def path(self, *parts):
"""A factory for :class:`RemotePaths <plumbum.path.remote.RemotePath>`.
Usage: ``p = rem.path("/usr", "lib", "python2.7")``
"""
parts2 = [str(self.cwd)]
for p in parts:
if isinstance(p, LocalPath):
raise TypeError(f"Cannot construct RemotePath from {p!r}")
parts2.append(self.expanduser(str(p)))
return RemotePath(self, *parts2)
def which(self, progname):
"""Looks up a program in the ``PATH``. If the program is not found, raises
:class:`CommandNotFound <plumbum.commands.CommandNotFound>`
:param progname: The program's name. Note that if underscores (``_``) are present
in the name, and the exact name is not found, they will be replaced
in turn by hyphens (``-``) then periods (``.``), and the name will
be looked up again for each alternative
:returns: A :class:`RemotePath <plumbum.path.local.RemotePath>`
"""
key = (progname, self.env.get("PATH", ""))
with contextlib.suppress(KeyError):
return self._program_cache[key]
alternatives = [progname]
if "_" in progname:
alternatives += [progname.replace("_", "-"), progname.replace("_", ".")]
for name in alternatives:
for p in self.env.path:
fn = p / name
if fn.access("x") and not fn.is_dir():
self._program_cache[key] = fn
return fn
raise CommandNotFound(progname, self.env.path)
def __getitem__(self, cmd):
"""Returns a `Command` object representing the given program. ``cmd`` can be a string or
a :class:`RemotePath <plumbum.path.remote.RemotePath>`; if it is a path, a command
representing this path will be returned; otherwise, the program name will be looked up in
the system's ``PATH`` (using ``which``). Usage::
r_ls = rem["ls"]
"""
if isinstance(cmd, RemotePath):
if cmd.remote is self:
return self.RemoteCommand(self, cmd)
raise TypeError(
f"Given path does not belong to this remote machine: {cmd!r}"
)
if not isinstance(cmd, LocalPath):
return self.RemoteCommand(
self, self.path(cmd) if "/" in cmd or "\\" in cmd else self.which(cmd)
)
raise TypeError(f"cmd must not be a LocalPath: {cmd!r}")
@property
def python(self):
"""A command that represents the default remote python interpreter"""
if not self._python:
self._python = self["python3"]
return self._python
def session(self, isatty=False, *, new_session=False):
"""Creates a new :class:`ShellSession <plumbum.session.ShellSession>` object; this invokes the user's
shell on the remote machine and executes commands on it over stdin/stdout/stderr"""
raise NotImplementedError()
def download(self, src, dst):
"""Downloads a remote file/directory (``src``) to a local destination (``dst``).
``src`` must be a string or a :class:`RemotePath <plumbum.path.remote.RemotePath>`
pointing to this remote machine, and ``dst`` must be a string or a
:class:`LocalPath <plumbum.machines.local.LocalPath>`"""
raise NotImplementedError()
def upload(self, src, dst):
"""Uploads a local file/directory (``src``) to a remote destination (``dst``).
``src`` must be a string or a :class:`LocalPath <plumbum.machines.local.LocalPath>`,
and ``dst`` must be a string or a :class:`RemotePath <plumbum.path.remote.RemotePath>`
pointing to this remote machine"""
raise NotImplementedError()
def popen(self, args, **kwargs):
"""Spawns the given command on the remote machine, returning a ``Popen``-like object;
do not use this method directly, unless you need "low-level" control on the remote
process"""
raise NotImplementedError()
def list_processes(self):
"""
Returns information about all running processes (on POSIX systems: using ``ps``)
.. versionadded:: 1.3
"""
ps = self["ps"]
lines = ps("-e", "-o", "pid,uid,stat,args").splitlines()
lines.pop(0) # header
for line in lines:
parts = line.strip().split()
yield ProcInfo(int(parts[0]), int(parts[1]), parts[2], " ".join(parts[3:]))
def pgrep(self, pattern):
"""
Process grep: return information about all processes whose command-line args match the given regex pattern
"""
pat = re.compile(pattern)
for procinfo in self.list_processes():
if pat.search(procinfo.args):
yield procinfo
@contextlib.contextmanager
def tempdir(self):
"""A context manager that creates a remote temporary directory, which is removed when
the context exits"""
_, out, _ = self._session.run(
"mktemp -d 2>/dev/null || mktemp -d tmp.XXXXXXXXXX"
)
local_dir = self.path(out.strip())
try:
yield local_dir
finally:
local_dir.delete()
#
# Path implementation
#
def _path_listdir(self, fn):
files = self._session.run(f"ls -a {shquote(fn)}")[1].splitlines()
files.remove(".")
files.remove("..")
return files
def _path_glob(self, fn, pattern):
# shquote does not work here due to the way bash loops use space as a separator
pattern = pattern.replace(" ", r"\ ")
fn = fn.replace(" ", r"\ ")
matches = self._session.run(rf"for fn in {fn}/{pattern}; do echo $fn; done")[
1
].splitlines()
if len(matches) == 1 and not self._path_stat(matches[0]):
return [] # pattern expansion failed
return matches
def _path_getuid(self, fn):
stat_cmd = (
"stat -c '%u,%U' "
if self.uname not in ("Darwin", "FreeBSD")
else "stat -f '%u,%Su' "
)
return self._session.run(stat_cmd + shquote(fn))[1].strip().split(",")
def _path_getgid(self, fn):
stat_cmd = (
"stat -c '%g,%G' "
if self.uname not in ("Darwin", "FreeBSD")
else "stat -f '%g,%Sg' "
)
return self._session.run(stat_cmd + shquote(fn))[1].strip().split(",")
def _path_stat(self, fn):
if self.uname not in ("Darwin", "FreeBSD"):
stat_cmd = "stat -c '%F,%f,%i,%d,%h,%u,%g,%s,%X,%Y,%Z' "
else:
stat_cmd = "stat -f '%HT,%Xp,%i,%d,%l,%u,%g,%z,%a,%m,%c' "
rc, out, _ = self._session.run(stat_cmd + shquote(fn), retcode=None)
if rc != 0:
return None
statres = out.strip().split(",")
text_mode = statres.pop(0).lower()
res = StatRes((int(statres[0], 16),) + tuple(int(sr) for sr in statres[1:]))
res.text_mode = text_mode
return res
def _path_delete(self, fn):
self._session.run(f"rm -rf {shquote(fn)}")
def _path_move(self, src, dst):
self._session.run(f"mv {shquote(src)} {shquote(dst)}")
def _path_copy(self, src, dst):
self._session.run(f"cp -r {shquote(src)} {shquote(dst)}")
def _path_mkdir(
self,
fn,
mode=None, # pylint: disable=unused-argument
minus_p=True,
):
p_str = "-p " if minus_p else ""
cmd = f"mkdir {p_str}{shquote(fn)}"
self._session.run(cmd)
def _path_chmod(self, mode, fn):
self._session.run(f"chmod {mode:o} {shquote(fn)}")
def _path_touch(self, path):
self._session.run(f"touch {path}")
def _path_chown(self, fn, owner, group, recursive):
args = ["chown"]
if recursive:
args.append("-R")
if owner is not None and group is not None:
args.append(f"{owner}:{group}")
elif owner is not None:
args.append(str(owner))
elif group is not None:
args.append(f":{group}")
args.append(shquote(fn))
self._session.run(" ".join(args))
def _path_read(self, fn):
data = self["cat"](fn)
if self.custom_encoding and isinstance(data, str):
data = data.encode(self.custom_encoding)
return data
def _path_write(self, fn, data):
if self.custom_encoding and isinstance(data, str):
data = data.encode(self.custom_encoding)
with NamedTemporaryFile() as f:
f.write(data)
f.flush()
f.seek(0)
self.upload(f.name, fn)
def _path_link(self, src, dst, symlink):
symlink_str = "-s " if symlink else ""
self._session.run(f"ln {symlink_str}{shquote(src)} {shquote(dst)}")
def expand(self, expr):
return self._session.run(f"echo {expr}")[1].strip()
def expanduser(self, expr):
if not any(part.startswith("~") for part in expr.split("/")):
return expr
# we escape all $ signs to avoid expanding env-vars
expr_repl = expr.replace("$", "\\$")
return self._session.run(f"echo {expr_repl}")[1].strip()

View File

@@ -0,0 +1,319 @@
import contextlib
import logging
import random
import threading
import time
from plumbum.commands import BaseCommand, run_proc
from plumbum.commands.processes import ProcessExecutionError
from plumbum.machines.base import PopenAddons
class ShellSessionError(Exception):
"""Raises when something goes wrong when calling
:func:`ShellSession.popen <plumbum.session.ShellSession.popen>`"""
class SSHCommsError(ProcessExecutionError, EOFError):
"""Raises when the communication channel can't be created on the
remote host or it times out."""
class SSHCommsChannel2Error(SSHCommsError):
"""Raises when channel 2 (stderr) is not available"""
class IncorrectLogin(SSHCommsError):
"""Raises when incorrect login credentials are provided"""
class HostPublicKeyUnknown(SSHCommsError):
"""Raises when the host public key isn't known"""
shell_logger = logging.getLogger("plumbum.shell")
# ===================================================================================================
# Shell Session Popen
# ===================================================================================================
class MarkedPipe:
"""A pipe-like object from which you can read lines; the pipe will return report EOF (the
empty string) when a special marker is detected"""
__slots__ = ["pipe", "marker", "__weakref__"]
def __init__(self, pipe, marker):
self.pipe = pipe
self.marker = marker
self.marker = bytes(self.marker, "ascii")
def close(self):
"""'Closes' the marked pipe; following calls to ``readline`` will return """ ""
# consume everything
while self.readline():
pass
self.pipe = None
def readline(self):
"""Reads the next line from the pipe; returns "" when the special marker is reached.
Raises ``EOFError`` if the underlying pipe has closed"""
if self.pipe is None:
return b""
line = self.pipe.readline()
if not line:
raise EOFError()
if line.strip() == self.marker:
self.pipe = None
line = b""
return line
class SessionPopen(PopenAddons):
"""A shell-session-based ``Popen``-like object (has the following attributes: ``stdin``,
``stdout``, ``stderr``, ``returncode``)"""
def __init__(self, proc, argv, isatty, stdin, stdout, stderr, encoding, *, host):
self.host = host
self.proc = proc
self.argv = argv
self.isatty = isatty
self.stdin = stdin
self.stdout = stdout
self.stderr = stderr
self.custom_encoding = encoding
self.returncode = None
self._done = False
def poll(self):
"""Returns the process' exit code or ``None`` if it's still running"""
return self.returncode if self._done else None
def wait(self):
"""Waits for the process to terminate and returns its exit code"""
self.communicate()
return self.returncode
def communicate(self, input=None): # pylint: disable=redefined-builtin
"""Consumes the process' stdout and stderr until the it terminates.
:param input: An optional bytes/buffer object to send to the process over stdin
:returns: A tuple of (stdout, stderr)
"""
stdout = []
stderr = []
sources = [("1", stdout, self.stdout)]
if not self.isatty:
# in tty mode, stdout and stderr are unified
sources.append(("2", stderr, self.stderr))
i = 0
while sources:
if input:
chunk = input[:1000]
self.stdin.write(chunk)
self.stdin.flush()
input = input[1000:]
i = (i + 1) % len(sources)
name, coll, pipe = sources[i]
try:
line = pipe.readline()
shell_logger.debug("%s> %r", name, line)
except EOFError as err:
shell_logger.debug("%s> Nothing returned.", name)
self.proc.poll()
returncode = self.proc.returncode
stdout = b"".join(stdout).decode(self.custom_encoding, "ignore")
stderr = b"".join(stderr).decode(self.custom_encoding, "ignore")
argv = self.argv.decode(self.custom_encoding, "ignore").split(";")[:1]
if returncode == 5:
raise IncorrectLogin(
argv,
returncode,
stdout,
stderr,
message="Incorrect username or password provided",
host=self.host,
) from None
if returncode == 6:
raise HostPublicKeyUnknown(
argv,
returncode,
stdout,
stderr,
message="The authenticity of the host can't be established",
host=self.host,
) from None
if returncode != 0:
raise SSHCommsError(
argv,
returncode,
stdout,
stderr,
message="SSH communication failed",
host=self.host,
) from None
if name == "2":
raise SSHCommsChannel2Error(
argv,
returncode,
stdout,
stderr,
message="No stderr result detected. Does the remote have Bash as the default shell?",
host=self.host,
) from None
raise SSHCommsError(
argv,
returncode,
stdout,
stderr,
message="No communication channel detected. Does the remote exist?",
host=self.host,
) from err
if not line:
del sources[i]
else:
coll.append(line)
if self.isatty:
stdout.pop(0) # discard first line of prompt
try:
self.returncode = int(stdout.pop(-1))
except (IndexError, ValueError):
self.returncode = "Unknown"
self._done = True
stdout = b"".join(stdout)
stderr = b"".join(stderr)
return stdout, stderr
class ShellSession:
"""An abstraction layer over *shell sessions*. A shell session is the execution of an
interactive shell (``/bin/sh`` or something compatible), over which you may run commands
(sent over stdin). The output of is then read from stdout and stderr. Shell sessions are
less "robust" than executing a process on its own, and they are susseptible to all sorts
of malformatted-strings attacks, and there is little benefit from using them locally.
However, they can greatly speed up remote connections, and are required for the implementation
of :class:`SshMachine <plumbum.machines.remote.SshMachine>`, as they allow us to send multiple
commands over a single SSH connection (setting up separate SSH connections incurs a high
overhead). Try to avoid using shell sessions, unless you know what you're doing.
Instances of this class may be used as *context-managers*.
:param proc: The underlying shell process (with open stdin, stdout and stderr)
:param encoding: The encoding to use for the shell session. If ``"auto"``, the underlying
process' encoding is used.
:param isatty: If true, assume the shell has a TTY and that stdout and stderr are unified
:param connect_timeout: The timeout to connect to the shell, after which, if no prompt
is seen, the shell process is killed
"""
def __init__(
self, proc, encoding="auto", isatty=False, connect_timeout=5, *, host=None
):
self.host = host
self.proc = proc
self.custom_encoding = proc.custom_encoding if encoding == "auto" else encoding
self.isatty = isatty
self._lock = threading.RLock()
self._current = None
self._startup_result = None
if connect_timeout:
def closer():
shell_logger.error(
"Connection to %s timed out (%d sec)", proc, connect_timeout
)
self.close()
timer = threading.Timer(connect_timeout, closer)
timer.start()
try:
self._startup_result = self.run("")
finally:
if connect_timeout:
timer.cancel()
def __enter__(self):
return self
def __exit__(self, t, v, tb):
self.close()
def __del__(self):
with contextlib.suppress(Exception):
self.close()
def alive(self):
"""Returns ``True`` if the underlying shell process is alive, ``False`` otherwise"""
return self.proc and self.proc.poll() is None
def close(self):
"""Closes (terminates) the shell session"""
if not self.alive():
return
with contextlib.suppress(ValueError, OSError):
self.proc.stdin.write(b"\nexit\n\n\nexit\n\n")
self.proc.stdin.flush()
time.sleep(0.05)
for p in (self.proc.stdin, self.proc.stdout, self.proc.stderr):
with contextlib.suppress(Exception):
p.close()
with contextlib.suppress(OSError):
self.proc.kill()
self.proc = None
def popen(self, cmd):
"""Runs the given command in the shell, adding some decoration around it. Only a single
command can be executed at any given time.
:param cmd: The command (string or :class:`Command <plumbum.commands.BaseCommand>` object)
to run
:returns: A :class:`SessionPopen <plumbum.session.SessionPopen>` instance
"""
if self.proc is None:
raise ShellSessionError("Shell session has already been closed")
if self._current and not self._current._done:
raise ShellSessionError("Each shell may start only one process at a time")
if isinstance(cmd, BaseCommand):
full_cmd = cmd.formulate(1)
else:
full_cmd = cmd
marker = f"--.END{time.time() * random.random()}.--"
if full_cmd.strip():
full_cmd += " ; "
else:
full_cmd = "true ; "
full_cmd += f"echo $? ; echo '{marker}'"
if not self.isatty:
full_cmd += f" ; echo '{marker}' 1>&2"
if self.custom_encoding:
full_cmd = full_cmd.encode(self.custom_encoding)
shell_logger.debug("Running %r", full_cmd)
self.proc.stdin.write(full_cmd + b"\n")
self.proc.stdin.flush()
self._current = SessionPopen(
self.proc,
full_cmd,
self.isatty,
self.proc.stdin,
MarkedPipe(self.proc.stdout, marker),
MarkedPipe(self.proc.stderr, marker),
self.custom_encoding,
host=self.host,
)
return self._current
def run(self, cmd, retcode=0):
"""Runs the given command
:param cmd: The command (string or :class:`Command <plumbum.commands.BaseCommand>` object)
to run
:param retcode: The expected return code (0 by default). Set to ``None`` in order to
ignore erroneous return codes
:returns: A tuple of (return code, stdout, stderr)
"""
with self._lock:
return run_proc(self.popen(cmd), retcode)

View File

@@ -0,0 +1,429 @@
import re
import socket
import warnings
from contextlib import closing
from plumbum.commands import ProcessExecutionError, shquote
from plumbum.lib import IS_WIN32
from plumbum.machines.local import local
from plumbum.machines.remote import BaseRemoteMachine
from plumbum.machines.session import ShellSession
from plumbum.path.local import LocalPath
from plumbum.path.remote import RemotePath
def _get_free_port():
"""Attempts to find a free port."""
s = socket.socket()
with closing(s):
s.bind(("localhost", 0))
return s.getsockname()[1]
class SshTunnel:
"""An object representing an SSH tunnel (created by
:func:`SshMachine.tunnel <plumbum.machines.remote.SshMachine.tunnel>`)"""
__slots__ = ["_session", "_lport", "_dport", "_reverse", "__weakref__"]
def __init__(self, session, lport, dport, reverse):
self._session = session
self._lport = lport
self._dport = dport
self._reverse = reverse
if reverse and str(dport) == "0" and session._startup_result is not None:
# Try to detect assigned remote port.
regex = re.compile(
r"^Allocated port (\d+) for remote forward to .+$", re.MULTILINE
)
match = regex.search(session._startup_result[2])
if match:
self._dport = match.group(1)
def __repr__(self):
tunnel = self._session.proc if self._session.alive() else "(defunct)"
return f"<SshTunnel {tunnel}>"
def __enter__(self):
return self
def __exit__(self, t, v, tb):
self.close()
def close(self):
"""Closes(terminates) the tunnel"""
self._session.close()
@property
def lport(self):
"""Tunneled port or socket on the local machine."""
return self._lport
@property
def dport(self):
"""Tunneled port or socket on the remote machine."""
return self._dport
@property
def reverse(self):
"""Represents if the tunnel is a reverse tunnel."""
return self._reverse
class SshMachine(BaseRemoteMachine):
"""
An implementation of :class:`remote machine <plumbum.machines.remote.BaseRemoteMachine>`
over SSH. Invoking a remote command translates to invoking it over SSH ::
with SshMachine("yourhostname") as rem:
r_ls = rem["ls"]
# r_ls is the remote `ls`
# executing r_ls() translates to `ssh yourhostname ls`
:param host: the host name to connect to (SSH server)
:param user: the user to connect as (if ``None``, the default will be used)
:param port: the server's port (if ``None``, the default will be used)
:param keyfile: the path to the identity file (if ``None``, the default will be used)
:param ssh_command: the ``ssh`` command to use; this has to be a ``Command`` object;
if ``None``, the default ssh client will be used.
:param scp_command: the ``scp`` command to use; this has to be a ``Command`` object;
if ``None``, the default scp program will be used.
:param ssh_opts: any additional options for ``ssh`` (a list of strings)
:param scp_opts: any additional options for ``scp`` (a list of strings)
:param password: the password to use; requires ``sshpass`` be installed. Cannot be used
in conjunction with ``ssh_command`` or ``scp_command`` (will be ignored).
NOTE: THIS IS A SECURITY RISK!
:param encoding: the remote machine's encoding (defaults to UTF8)
:param connect_timeout: specify a connection timeout (the time until shell prompt is seen).
The default is 10 seconds. Set to ``None`` to disable
:param new_session: whether or not to start the background session as a new
session leader (setsid). This will prevent it from being killed on
Ctrl+C (SIGINT)
"""
def __init__(
self,
host,
user=None,
port=None,
keyfile=None,
ssh_command=None,
scp_command=None,
ssh_opts=(),
scp_opts=(),
password=None,
encoding="utf8",
connect_timeout=10,
new_session=False,
):
if ssh_command is None:
if password is not None:
ssh_command = local["sshpass"]["-p", password, "ssh"]
else:
ssh_command = local["ssh"]
if scp_command is None:
if password is not None:
scp_command = local["sshpass"]["-p", password, "scp"]
else:
scp_command = local["scp"]
scp_args = []
ssh_args = []
self.host = host
if user:
self._fqhost = f"{user}@{host}"
else:
self._fqhost = host
if port:
ssh_args.extend(["-p", str(port)])
scp_args.extend(["-P", str(port)])
if keyfile:
ssh_args.extend(["-i", str(keyfile)])
scp_args.extend(["-i", str(keyfile)])
scp_args.append("-r")
ssh_args.extend(ssh_opts)
scp_args.extend(scp_opts)
self._ssh_command = ssh_command[tuple(ssh_args)]
self._scp_command = scp_command[tuple(scp_args)]
BaseRemoteMachine.__init__(
self,
encoding=encoding,
connect_timeout=connect_timeout,
new_session=new_session,
)
def __str__(self):
return f"ssh://{self._fqhost}"
def popen(self, args, ssh_opts=(), env=None, cwd=None, **kwargs):
cmdline = []
cmdline.extend(ssh_opts)
cmdline.append(self._fqhost)
if args:
envdelta = {}
if hasattr(self, "env"):
envdelta.update(self.env.getdelta())
if env:
envdelta.update(env)
if cwd is None:
cwd = getattr(self, "cwd", None)
if cwd:
cmdline.extend(["cd", str(cwd), "&&"])
if envdelta:
cmdline.append("env")
cmdline.extend(f"{k}={shquote(v)}" for k, v in envdelta.items())
if isinstance(args, (tuple, list)):
cmdline.extend(args)
else:
cmdline.append(args)
return self._ssh_command[tuple(cmdline)].popen(**kwargs)
def nohup(self, command):
"""
Runs the given command using ``nohup`` and redirects std handles,
allowing the command to run "detached" from its controlling TTY or parent.
Does not return anything. Depreciated (use command.nohup or daemonic_popen).
"""
warnings.warn("Use .nohup on the command or use daemonic_popen)", FutureWarning)
self.daemonic_popen(command, cwd=".", stdout=None, stderr=None, append=False)
def daemonic_popen(self, command, cwd=".", stdout=None, stderr=None, append=True):
"""
Runs the given command using ``nohup`` and redirects std handles,
allowing the command to run "detached" from its controlling TTY or parent.
Does not return anything.
.. versionadded:: 1.6.0
"""
if stdout is None:
stdout = "/dev/null"
if stderr is None:
stderr = "&1"
if str(cwd) == ".":
args = []
else:
args = ["cd", str(cwd), "&&"]
args.append("nohup")
args.extend(command.formulate())
args.extend(
[
(">>" if append else ">") + str(stdout),
"2" + (">>" if (append and stderr != "&1") else ">") + str(stderr),
"</dev/null",
]
)
proc = self.popen(args, ssh_opts=["-f"])
rc = proc.wait()
try:
if rc != 0:
raise ProcessExecutionError(
args, rc, proc.stdout.read(), proc.stderr.read()
)
finally:
proc.stdin.close()
proc.stdout.close()
proc.stderr.close()
def session(self, isatty=False, new_session=False):
return ShellSession(
self.popen(
["/bin/sh"], (["-tt"] if isatty else ["-T"]), new_session=new_session
),
self.custom_encoding,
isatty,
self.connect_timeout,
host=self.host,
)
def tunnel(
self,
lport,
dport,
lhost="localhost",
dhost="localhost",
connect_timeout=5, # pylint: disable=unused-argument
reverse=False,
):
r"""Creates an SSH tunnel from the TCP port (``lport``) of the local machine
(``lhost``, defaults to ``"localhost"``, but it can be any IP you can ``bind()``)
to the remote TCP port (``dport``) of the destination machine (``dhost``, defaults
to ``"localhost"``, which means *this remote machine*). This function also
supports Unix sockets, in which case the local socket should be passed in as
``lport`` and the local bind address should be ``None``. The same can be done
for a remote socket, by following the same pattern with ``dport`` and ``dhost``.
The returned :class:`SshTunnel <plumbum.machines.remote.SshTunnel>` object can
be used as a *context-manager*.
The more conventional use case is the following::
+---------+ +---------+
| Your | | Remote |
| Machine | | Machine |
+----o----+ +---- ----+
| ^
| |
lport dport
| |
\______SSH TUNNEL____/
(secure)
Here, you wish to communicate safely between port ``lport`` of your machine and
port ``dport`` of the remote machine. Communication is tunneled over SSH, so the
connection is authenticated and encrypted.
The more general case is shown below (where ``dport != "localhost"``)::
+---------+ +-------------+ +-------------+
| Your | | Remote | | Destination |
| Machine | | Machine | | Machine |
+----o----+ +---- ----o---+ +---- --------+
| ^ | ^
| | | |
lhost:lport | | dhost:dport
| | | |
\_____SSH TUNNEL_____/ \_____SOCKET____/
(secure) (not secure)
Usage::
rem = SshMachine("megazord")
with rem.tunnel(1234, "/var/lib/mysql/mysql.sock", dhost=None):
sock = socket.socket()
sock.connect(("localhost", 1234))
# sock is now tunneled to the MySQL socket on megazord
"""
formatted_lhost = "" if lhost is None else f"[{lhost}]:"
formatted_dhost = "" if dhost is None else f"[{dhost}]:"
if str(lport) == "0":
lport = _get_free_port()
ssh_opts = (
[
"-L",
f"{formatted_lhost}{lport}:{formatted_dhost}{dport}",
]
if not reverse
else [
"-R",
f"{formatted_dhost}{dport}:{formatted_lhost}{lport}",
]
)
proc = self.popen((), ssh_opts=ssh_opts, new_session=True)
return SshTunnel(
ShellSession(
proc, self.custom_encoding, connect_timeout=self.connect_timeout
),
lport,
dport,
reverse,
)
def _translate_drive_letter(self, path): # pylint: disable=no-self-use
# replace c:\some\path with /c/some/path
path = str(path)
if ":" in path:
path = "/" + path.replace(":", "").replace("\\", "/")
return path
def download(self, src, dst):
if isinstance(src, LocalPath):
raise TypeError(f"src of download cannot be {src!r}")
if isinstance(src, RemotePath) and src.remote != self:
raise TypeError(f"src {src!r} points to a different remote machine")
if isinstance(dst, RemotePath):
raise TypeError(f"dst of download cannot be {dst!r}")
if IS_WIN32:
src = self._translate_drive_letter(src)
dst = self._translate_drive_letter(dst)
self._scp_command(f"{self._fqhost}:{shquote(src)}", dst)
def upload(self, src, dst):
if isinstance(src, RemotePath):
raise TypeError(f"src of upload cannot be {src!r}")
if isinstance(dst, LocalPath):
raise TypeError(f"dst of upload cannot be {dst!r}")
if isinstance(dst, RemotePath) and dst.remote != self:
raise TypeError(f"dst {dst!r} points to a different remote machine")
if IS_WIN32:
src = self._translate_drive_letter(src)
dst = self._translate_drive_letter(dst)
self._scp_command(src, f"{self._fqhost}:{shquote(dst)}")
class PuttyMachine(SshMachine):
"""
PuTTY-flavored SSH connection. The programs ``plink`` and ``pscp`` are expected to
be in the path (or you may provide your own ``ssh_command`` and ``scp_command``)
Arguments are the same as for :class:`plumbum.machines.remote.SshMachine`
"""
def __init__(
self,
host,
user=None,
port=None,
keyfile=None,
ssh_command=None,
scp_command=None,
ssh_opts=(),
scp_opts=(),
encoding="utf8",
connect_timeout=10,
new_session=False,
):
if ssh_command is None:
ssh_command = local["plink"]
if scp_command is None:
scp_command = local["pscp"]
if not ssh_opts:
ssh_opts = ["-ssh"]
if user is None:
user = local.env.user
if port is not None:
ssh_opts.extend(["-P", str(port)])
scp_opts = list(scp_opts) + ["-P", str(port)]
port = None
SshMachine.__init__(
self,
host,
user,
port,
keyfile=keyfile,
ssh_command=ssh_command,
scp_command=scp_command,
ssh_opts=ssh_opts,
scp_opts=scp_opts,
encoding=encoding,
connect_timeout=connect_timeout,
new_session=new_session,
)
def __str__(self):
return f"putty-ssh://{self._fqhost}"
def _translate_drive_letter(self, path):
# pscp takes care of windows paths automatically
return path
def session(self, isatty=False, new_session=False):
return ShellSession(
self.popen((), (["-t"] if isatty else ["-T"]), new_session=new_session),
self.custom_encoding,
isatty,
self.connect_timeout,
)

View File

@@ -0,0 +1,17 @@
from plumbum.path.base import FSUser, Path, RelativePath
from plumbum.path.local import LocalPath, LocalWorkdir
from plumbum.path.remote import RemotePath, RemoteWorkdir
from plumbum.path.utils import copy, delete, move
__all__ = (
"FSUser",
"Path",
"RelativePath",
"LocalPath",
"LocalWorkdir",
"RemotePath",
"RemoteWorkdir",
"copy",
"delete",
"move",
)

View File

@@ -0,0 +1,494 @@
import itertools
import operator
import os
import warnings
from abc import ABC, abstractmethod
from functools import reduce
FLAGS = {"f": os.F_OK, "w": os.W_OK, "r": os.R_OK, "x": os.X_OK}
class FSUser(int):
"""A special object that represents a file-system user. It derives from ``int``, so it behaves
just like a number (``uid``/``gid``), but also have a ``.name`` attribute that holds the
string-name of the user, if given (otherwise ``None``)
"""
def __new__(cls, val, name=None):
self = int.__new__(cls, val)
self.name = name
return self
class Path(str, ABC):
"""An abstraction over file system paths. This class is abstract, and the two implementations
are :class:`LocalPath <plumbum.machines.local.LocalPath>` and
:class:`RemotePath <plumbum.path.remote.RemotePath>`.
"""
CASE_SENSITIVE = True
def __repr__(self):
return f"<{self.__class__.__name__} {self}>"
def __truediv__(self, other):
"""Joins two paths"""
return self.join(other)
def __getitem__(self, key):
if type(key) == str or isinstance(key, Path):
return self / key
return str(self)[key]
def __floordiv__(self, expr):
"""Returns a (possibly empty) list of paths that matched the glob-pattern under this path"""
return self.glob(expr)
def __iter__(self):
"""Iterate over the files in this directory"""
return iter(self.list())
def __eq__(self, other):
if isinstance(other, Path):
return self._get_info() == other._get_info()
if isinstance(other, str):
if self.CASE_SENSITIVE:
return str(self) == other
return str(self).lower() == other.lower()
return NotImplemented
def __ne__(self, other):
return not self == other
def __gt__(self, other):
return str(self) > str(other)
def __ge__(self, other):
return str(self) >= str(other)
def __lt__(self, other):
return str(self) < str(other)
def __le__(self, other):
return str(self) <= str(other)
def __hash__(self):
return hash(str(self)) if self.CASE_SENSITIVE else hash(str(self).lower())
def __bool__(self):
return bool(str(self))
def __fspath__(self):
"""Added for Python 3.6 support"""
return str(self)
def __contains__(self, item):
"""Paths should support checking to see if an file or folder is in them."""
try:
return (self / item.name).exists()
except AttributeError:
return (self / item).exists()
@abstractmethod
def _form(self, *parts):
pass
def up(self, count=1):
"""Go up in ``count`` directories (the default is 1)"""
return self.join("../" * count)
def walk(
self,
filter=lambda p: True, # pylint: disable=redefined-builtin
dir_filter=lambda p: True,
):
"""traverse all (recursive) sub-elements under this directory, that match the given filter.
By default, the filter accepts everything; you can provide a custom filter function that
takes a path as an argument and returns a boolean
:param filter: the filter (predicate function) for matching results. Only paths matching
this predicate are returned. Defaults to everything.
:param dir_filter: the filter (predicate function) for matching directories. Only directories
matching this predicate are recursed into. Defaults to everything.
"""
for p in self.list():
if filter(p):
yield p
if p.is_dir() and dir_filter(p):
yield from p.walk(filter, dir_filter)
@property
@abstractmethod
def name(self):
"""The basename component of this path"""
@property
def basename(self):
"""Included for compatibility with older Plumbum code"""
warnings.warn("Use .name instead", FutureWarning)
return self.name
@property
@abstractmethod
def stem(self):
"""The name without an extension, or the last component of the path"""
@property
@abstractmethod
def dirname(self):
"""The dirname component of this path"""
@property
@abstractmethod
def root(self):
"""The root of the file tree (`/` on Unix)"""
@property
@abstractmethod
def drive(self):
"""The drive letter (on Windows)"""
@property
@abstractmethod
def suffix(self):
"""The suffix of this file"""
@property
@abstractmethod
def suffixes(self):
"""This is a list of all suffixes"""
@property
@abstractmethod
def uid(self):
"""The user that owns this path. The returned value is a :class:`FSUser <plumbum.path.FSUser>`
object which behaves like an ``int`` (as expected from ``uid``), but it also has a ``.name``
attribute that holds the string-name of the user"""
@property
@abstractmethod
def gid(self):
"""The group that owns this path. The returned value is a :class:`FSUser <plumbum.path.FSUser>`
object which behaves like an ``int`` (as expected from ``gid``), but it also has a ``.name``
attribute that holds the string-name of the group"""
@abstractmethod
def as_uri(self, scheme=None):
"""Returns a universal resource identifier. Use ``scheme`` to force a scheme."""
@abstractmethod
def _get_info(self):
pass
@abstractmethod
def join(self, *parts):
"""Joins this path with any number of paths"""
@abstractmethod
def list(self):
"""Returns the files in this directory"""
@abstractmethod
def iterdir(self):
"""Returns an iterator over the directory. Might be slightly faster on Python 3.5 than .list()"""
@abstractmethod
def is_dir(self):
"""Returns ``True`` if this path is a directory, ``False`` otherwise"""
def isdir(self):
"""Included for compatibility with older Plumbum code"""
warnings.warn("Use .is_dir() instead", FutureWarning)
return self.is_dir()
@abstractmethod
def is_file(self):
"""Returns ``True`` if this path is a regular file, ``False`` otherwise"""
def isfile(self):
"""Included for compatibility with older Plumbum code"""
warnings.warn("Use .is_file() instead", FutureWarning)
return self.is_file()
def islink(self):
"""Included for compatibility with older Plumbum code"""
warnings.warn("Use is_symlink instead", FutureWarning)
return self.is_symlink()
@abstractmethod
def is_symlink(self):
"""Returns ``True`` if this path is a symbolic link, ``False`` otherwise"""
@abstractmethod
def exists(self):
"""Returns ``True`` if this path exists, ``False`` otherwise"""
@abstractmethod
def stat(self):
"""Returns the os.stats for a file"""
@abstractmethod
def with_name(self, name):
"""Returns a path with the name replaced"""
@abstractmethod
def with_suffix(self, suffix, depth=1):
"""Returns a path with the suffix replaced. Up to last ``depth`` suffixes will be
replaced. None will replace all suffixes. If there are less than ``depth`` suffixes,
this will replace all suffixes. ``.tar.gz`` is an example where ``depth=2`` or
``depth=None`` is useful"""
def preferred_suffix(self, suffix):
"""Adds a suffix if one does not currently exist (otherwise, no change). Useful
for loading files with a default suffix"""
return self if len(self.suffixes) > 0 else self.with_suffix(suffix)
@abstractmethod
def glob(self, pattern):
"""Returns a (possibly empty) list of paths that matched the glob-pattern under this path"""
@abstractmethod
def delete(self):
"""Deletes this path (recursively, if a directory)"""
@abstractmethod
def move(self, dst):
"""Moves this path to a different location"""
def rename(self, newname):
"""Renames this path to the ``new name`` (only the basename is changed)"""
return self.move(self.up() / newname)
@abstractmethod
def copy(self, dst, override=None):
"""Copies this path (recursively, if a directory) to the destination path "dst".
Raises TypeError if dst exists and override is False.
Will overwrite if override is True.
Will silently fail to copy if override is None (the default)."""
@abstractmethod
def mkdir(self, mode=0o777, parents=True, exist_ok=True):
"""
Creates a directory at this path.
:param mode: **Currently only implemented for local paths!** Numeric mode to use for directory
creation, which may be ignored on some systems. The current implementation
reproduces the behavior of ``os.mkdir`` (i.e., the current umask is first masked
out), but this may change for remote paths. As with ``os.mkdir``, it is recommended
to call :func:`chmod` explicitly if you need to be sure.
:param parents: If this is true (the default), the directory's parents will also be created if
necessary.
:param exist_ok: If this is true (the default), no exception will be raised if the directory
already exists (otherwise ``OSError``).
Note that the defaults for ``parents`` and ``exist_ok`` are the opposite of what they are in
Python's own ``pathlib`` - this is to maintain backwards-compatibility with Plumbum's behaviour
from before they were implemented.
"""
@abstractmethod
def open(self, mode="r", *, encoding=None):
"""opens this path as a file"""
@abstractmethod
def read(self, encoding=None):
"""returns the contents of this file as a ``str``. By default the data is read
as text, but you can specify the encoding, e.g., ``'latin1'`` or ``'utf8'``"""
@abstractmethod
def write(self, data, encoding=None):
"""writes the given data to this file. By default the data is written as-is
(either text or binary), but you can specify the encoding, e.g., ``'latin1'``
or ``'utf8'``"""
@abstractmethod
def touch(self):
"""Update the access time. Creates an empty file if none exists."""
@abstractmethod
def chown(self, owner=None, group=None, recursive=None):
"""Change ownership of this path.
:param owner: The owner to set (either ``uid`` or ``username``), optional
:param group: The group to set (either ``gid`` or ``groupname``), optional
:param recursive: whether to change ownership of all contained files and subdirectories.
Only meaningful when ``self`` is a directory. If ``None``, the value
will default to ``True`` if ``self`` is a directory, ``False`` otherwise.
"""
@abstractmethod
def chmod(self, mode):
"""Change the mode of path to the numeric mode.
:param mode: file mode as for os.chmod
"""
@staticmethod
def _access_mode_to_flags(mode, flags=None):
if flags is None:
flags = FLAGS
if isinstance(mode, str):
mode = reduce(operator.or_, [flags[m] for m in mode.lower()], 0)
return mode
@abstractmethod
def access(self, mode=0):
"""Test file existence or permission bits
:param mode: a bitwise-or of access bits, or a string-representation thereof:
``'f'``, ``'x'``, ``'r'``, ``'w'`` for ``os.F_OK``, ``os.X_OK``,
``os.R_OK``, ``os.W_OK``
"""
@abstractmethod
def link(self, dst):
"""Creates a hard link from ``self`` to ``dst``
:param dst: the destination path
"""
@abstractmethod
def symlink(self, dst):
"""Creates a symbolic link from ``self`` to ``dst``
:param dst: the destination path
"""
@abstractmethod
def unlink(self):
"""Deletes a symbolic link"""
def split(self, *_args, **_kargs):
"""Splits the path on directory separators, yielding a list of directories, e.g,
``"/var/log/messages"`` will yield ``['var', 'log', 'messages']``.
"""
parts = []
path = self
while path != path.dirname:
parts.append(path.name)
path = path.dirname
return parts[::-1]
@property
def parts(self):
"""Splits the directory into parts, including the base directory, returns a tuple"""
return tuple([self.drive + self.root] + self.split())
def relative_to(self, source):
"""Computes the "relative path" require to get from ``source`` to ``self``. They satisfy the invariant
``source_path + (target_path - source_path) == target_path``. For example::
/var/log/messages - /var/log/messages = []
/var/log/messages - /var = [log, messages]
/var/log/messages - / = [var, log, messages]
/var/log/messages - /var/tmp = [.., log, messages]
/var/log/messages - /opt = [.., var, log, messages]
/var/log/messages - /opt/lib = [.., .., var, log, messages]
"""
if isinstance(source, str):
source = self._form(source)
parts = self.split()
baseparts = source.split()
ancestors = len(
list(itertools.takewhile(lambda p: p[0] == p[1], zip(parts, baseparts)))
)
return RelativePath([".."] * (len(baseparts) - ancestors) + parts[ancestors:])
def __sub__(self, other):
"""Same as ``self.relative_to(other)``"""
return self.relative_to(other)
@staticmethod
def _glob(pattern, fn):
"""Applies a glob string or list/tuple/iterable to the current path, using ``fn``"""
if isinstance(pattern, str):
return fn(pattern)
results = []
for single_pattern in pattern:
results.extend(fn(single_pattern))
return sorted(list(set(results)))
def resolve(self, strict=False): # pylint:disable=unused-argument
"""Added to allow pathlib like syntax. Does nothing since
Plumbum paths are always absolute. Does not (currently) resolve
symlinks."""
# TODO: Resolve symlinks here
return self
@property
def parents(self):
"""Pathlib like sequence of ancestors"""
as_list = (
reduce(lambda x, y: self._form(x) / y, self.parts[:i], self.parts[0])
for i in range(len(self.parts) - 1, 0, -1)
)
return tuple(as_list)
@property
def parent(self):
"""Pathlib like parent of the path."""
return self.parents[0]
class RelativePath:
"""
Relative paths are the "delta" required to get from one path to another.
Note that relative path do not point at anything, and thus are not paths.
Therefore they are system agnostic (but closed under addition)
Paths are always absolute and point at "something", whether existent or not.
Relative paths are created by subtracting paths (``Path.relative_to``)
"""
def __init__(self, parts):
self.parts = parts
def __str__(self):
return "/".join(self.parts)
def __iter__(self):
return iter(self.parts)
def __len__(self):
return len(self.parts)
def __getitem__(self, index):
return self.parts[index]
def __repr__(self):
return f"RelativePath({self.parts!r})"
def __eq__(self, other):
return str(self) == str(other)
def __ne__(self, other):
return not self == other
def __gt__(self, other):
return str(self) > str(other)
def __ge__(self, other):
return str(self) >= str(other)
def __lt__(self, other):
return str(self) < str(other)
def __le__(self, other):
return str(self) <= str(other)
def __hash__(self):
return hash(str(self))
def __bool__(self):
return bool(str(self))
def up(self, count=1):
return RelativePath(self.parts[:-count])
def __radd__(self, path):
return path.join(*self.parts)

View File

@@ -0,0 +1,355 @@
import errno
import glob
import logging
import os
import shutil
import urllib.parse as urlparse
import urllib.request as urllib
from contextlib import contextmanager
from plumbum.lib import IS_WIN32
from plumbum.path.base import FSUser, Path
from plumbum.path.remote import RemotePath
try:
from grp import getgrgid, getgrnam
from pwd import getpwnam, getpwuid
except ImportError:
def getpwuid(_x): # type: ignore[misc]
return (None,)
def getgrgid(_x): # type: ignore[misc]
return (None,)
def getpwnam(_x): # type: ignore[misc]
raise OSError("`getpwnam` not supported")
def getgrnam(_x): # type: ignore[misc]
raise OSError("`getgrnam` not supported")
logger = logging.getLogger("plumbum.local")
_EMPTY = object()
# ===================================================================================================
# Local Paths
# ===================================================================================================
class LocalPath(Path):
"""The class implementing local-machine paths"""
CASE_SENSITIVE = not IS_WIN32
def __new__(cls, *parts):
if (
len(parts) == 1
and isinstance(parts[0], cls)
and not isinstance(parts[0], LocalWorkdir)
):
return parts[0]
if not parts:
raise TypeError("At least one path part is required (none given)")
if any(isinstance(path, RemotePath) for path in parts):
raise TypeError(f"LocalPath cannot be constructed from {parts!r}")
self = super().__new__(
cls, os.path.normpath(os.path.join(*(str(p) for p in parts)))
)
return self
@property
def _path(self):
return str(self)
def _get_info(self):
return self._path
def _form(self, *parts):
return LocalPath(*parts)
@property
def name(self):
return os.path.basename(str(self))
@property
def dirname(self):
return LocalPath(os.path.dirname(str(self)))
@property
def suffix(self):
return os.path.splitext(str(self))[1]
@property
def suffixes(self):
exts = []
base = str(self)
while True:
base, ext = os.path.splitext(base)
if ext:
exts.append(ext)
else:
return list(reversed(exts))
@property
def uid(self):
uid = self.stat().st_uid
name = getpwuid(uid)[0]
return FSUser(uid, name)
@property
def gid(self):
gid = self.stat().st_gid
name = getgrgid(gid)[0]
return FSUser(gid, name)
def join(self, *others):
return LocalPath(self, *others)
def list(self):
return [self / fn for fn in os.listdir(str(self))]
def iterdir(self):
try:
return (self / fn.name for fn in os.scandir(str(self)))
except AttributeError:
return (self / fn for fn in os.listdir(str(self)))
def is_dir(self):
return os.path.isdir(str(self))
def is_file(self):
return os.path.isfile(str(self))
def is_symlink(self):
return os.path.islink(str(self))
def exists(self):
return os.path.exists(str(self))
def stat(self):
return os.stat(str(self))
def with_name(self, name):
return LocalPath(self.dirname) / name
@property
def stem(self):
return self.name.rsplit(os.path.extsep)[0]
def with_suffix(self, suffix, depth=1):
if suffix and not suffix.startswith(os.path.extsep) or suffix == os.path.extsep:
raise ValueError(f"Invalid suffix {suffix!r}")
name = self.name
depth = len(self.suffixes) if depth is None else min(depth, len(self.suffixes))
for _ in range(depth):
name, _ = os.path.splitext(name)
return LocalPath(self.dirname) / (name + suffix)
def glob(self, pattern):
return self._glob(
pattern,
lambda pat: [
LocalPath(m)
for m in glob.glob(os.path.join(glob.escape(str(self)), pat))
],
)
def delete(self):
if not self.exists():
return
if self.is_dir():
shutil.rmtree(str(self))
else:
try:
os.remove(str(self))
except OSError as ex: # pragma: no cover
# file might already been removed (a race with other threads/processes)
if ex.errno != errno.ENOENT:
raise
def move(self, dst):
if isinstance(dst, RemotePath):
raise TypeError(f"Cannot move local path {self} to {dst!r}")
shutil.move(str(self), str(dst))
return LocalPath(dst)
def copy(self, dst, override=None):
if isinstance(dst, RemotePath):
raise TypeError(f"Cannot copy local path {self} to {dst!r}")
dst = LocalPath(dst)
if override is False and dst.exists():
raise TypeError("File exists and override was not specified")
if override:
dst.delete()
if self.is_dir():
shutil.copytree(str(self), str(dst))
else:
dst_dir = LocalPath(dst).dirname
if not dst_dir.exists():
dst_dir.mkdir()
shutil.copy2(str(self), str(dst))
return dst
def mkdir(self, mode=0o777, parents=True, exist_ok=True):
if not self.exists() or not exist_ok:
try:
if parents:
os.makedirs(str(self), mode)
else:
os.mkdir(str(self), mode)
except OSError as ex: # pragma: no cover
# directory might already exist (a race with other threads/processes)
if ex.errno != errno.EEXIST or not exist_ok:
raise
def open(self, mode="r", encoding=None):
return open(
str(self),
mode,
encoding=encoding,
)
def read(self, encoding=None, mode="r"):
if encoding and "b" not in mode:
mode = mode + "b"
with self.open(mode) as f:
data = f.read()
if encoding:
data = data.decode(encoding)
return data
def write(self, data, encoding=None, mode=None):
if encoding:
data = data.encode(encoding)
if mode is None:
if isinstance(data, str):
mode = "w"
else:
mode = "wb"
with self.open(mode) as f:
f.write(data)
def touch(self):
with open(str(self), "a", encoding="utf-8"):
os.utime(str(self), None)
def chown(self, owner=None, group=None, recursive=None):
if not hasattr(os, "chown"):
raise OSError("os.chown() not supported")
uid = (
self.uid
if owner is None
else (owner if isinstance(owner, int) else getpwnam(owner)[2])
)
gid = (
self.gid
if group is None
else (group if isinstance(group, int) else getgrnam(group)[2])
)
os.chown(str(self), uid, gid)
if recursive or (recursive is None and self.is_dir()):
for subpath in self.walk():
os.chown(str(subpath), uid, gid)
def chmod(self, mode):
if not hasattr(os, "chmod"):
raise OSError("os.chmod() not supported")
os.chmod(str(self), mode)
def access(self, mode=0):
return os.access(str(self), self._access_mode_to_flags(mode))
def link(self, dst):
if isinstance(dst, RemotePath):
raise TypeError(
f"Cannot create a hardlink from local path {self} to {dst!r}"
)
if hasattr(os, "link"):
os.link(str(self), str(dst))
else:
from plumbum.machines.local import local
# windows: use mklink
if self.is_dir():
local["cmd"]("/C", "mklink", "/D", "/H", str(dst), str(self))
else:
local["cmd"]("/C", "mklink", "/H", str(dst), str(self))
def symlink(self, dst):
if isinstance(dst, RemotePath):
raise TypeError(
f"Cannot create a symlink from local path {self} to {dst!r}"
)
if hasattr(os, "symlink"):
os.symlink(str(self), str(dst))
else:
from plumbum.machines.local import local
# windows: use mklink
if self.is_dir():
local["cmd"]("/C", "mklink", "/D", str(dst), str(self))
else:
local["cmd"]("/C", "mklink", str(dst), str(self))
def unlink(self):
try:
if hasattr(os, "symlink") or not self.is_dir():
os.unlink(str(self))
else:
# windows: use rmdir for directories and directory symlinks
os.rmdir(str(self))
except OSError as ex: # pragma: no cover
# file might already been removed (a race with other threads/processes)
if ex.errno != errno.ENOENT:
raise
def as_uri(self, scheme="file"):
return urlparse.urljoin(str(scheme) + ":", urllib.pathname2url(str(self)))
@property
def drive(self):
return os.path.splitdrive(str(self))[0]
@property
def root(self):
return os.path.sep
class LocalWorkdir(LocalPath):
"""Working directory manipulator"""
def __hash__(self):
raise TypeError("unhashable type")
def __new__(cls):
return super().__new__(cls, os.getcwd())
def chdir(self, newdir):
"""Changes the current working directory to the given one
:param newdir: The destination director (a string or a ``LocalPath``)
"""
if isinstance(newdir, RemotePath):
raise TypeError(f"newdir cannot be {newdir!r}")
logger.debug("Chdir to %s", newdir)
os.chdir(str(newdir))
return self.__class__()
def getpath(self):
"""Returns the current working directory as a ``LocalPath`` object"""
return LocalPath(self._path)
@contextmanager
def __call__(self, newdir):
"""A context manager used to ``chdir`` into a directory and then ``chdir`` back to
the previous location; much like ``pushd``/``popd``.
:param newdir: The destination directory (a string or a ``LocalPath``)
"""
prev = self._path
newdir = self.chdir(newdir)
try:
yield newdir
finally:
self.chdir(prev)

View File

@@ -0,0 +1,357 @@
import errno
import os
import urllib.request as urllib
from contextlib import contextmanager
from plumbum.commands import ProcessExecutionError, shquote
from plumbum.path.base import FSUser, Path
class StatRes:
"""POSIX-like stat result"""
def __init__(self, tup):
self._tup = tuple(tup)
def __getitem__(self, index):
return self._tup[index]
st_mode = mode = property(lambda self: self[0])
st_ino = ino = property(lambda self: self[1])
st_dev = dev = property(lambda self: self[2])
st_nlink = nlink = property(lambda self: self[3])
st_uid = uid = property(lambda self: self[4])
st_gid = gid = property(lambda self: self[5])
st_size = size = property(lambda self: self[6])
st_atime = atime = property(lambda self: self[7])
st_mtime = mtime = property(lambda self: self[8])
st_ctime = ctime = property(lambda self: self[9])
class RemotePath(Path):
"""The class implementing remote-machine paths"""
def __new__(cls, remote, *parts):
if not parts:
raise TypeError("At least one path part is required (none given)")
windows = remote.uname.lower() == "windows"
normed = []
parts = tuple(
map(str, parts)
) # force the paths into string, so subscription works properly
# Simple skip if path is absolute
if parts[0] and parts[0][0] not in ("/", "\\"):
cwd = (
remote._cwd
if hasattr(remote, "_cwd")
else remote._session.run("pwd")[1].strip()
)
parts = (cwd,) + parts
for p in parts:
if windows:
plist = str(p).replace("\\", "/").split("/")
else:
plist = str(p).split("/")
if not plist[0]:
plist.pop(0)
del normed[:]
for item in plist:
if item in {"", "."}:
continue
if item == "..":
if normed:
normed.pop(-1)
else:
normed.append(item)
if windows:
self = super().__new__(cls, "\\".join(normed))
self.CASE_SENSITIVE = False # On this object only
else:
self = super().__new__(cls, "/" + "/".join(normed))
self.CASE_SENSITIVE = True
self.remote = remote
return self
def _form(self, *parts):
return RemotePath(self.remote, *parts)
@property
def _path(self):
return str(self)
@property
def name(self):
if "/" not in str(self):
return str(self)
return str(self).rsplit("/", 1)[1]
@property
def dirname(self):
if "/" not in str(self):
return str(self)
return self.__class__(self.remote, str(self).rsplit("/", 1)[0])
@property
def suffix(self):
return "." + self.name.rsplit(".", 1)[1]
@property
def suffixes(self):
name = self.name
exts = []
while "." in name:
name, ext = name.rsplit(".", 1)
exts.append("." + ext)
return list(reversed(exts))
@property
def uid(self):
uid, name = self.remote._path_getuid(self)
return FSUser(int(uid), name)
@property
def gid(self):
gid, name = self.remote._path_getgid(self)
return FSUser(int(gid), name)
def _get_info(self):
return (self.remote, self._path)
def join(self, *parts):
return RemotePath(self.remote, self, *parts)
def list(self):
if not self.is_dir():
return []
return [self.join(fn) for fn in self.remote._path_listdir(self)]
def iterdir(self):
if not self.is_dir():
return ()
return (self.join(fn) for fn in self.remote._path_listdir(self))
def is_dir(self):
res = self.remote._path_stat(self)
if not res:
return False
return res.text_mode == "directory"
def is_file(self):
res = self.remote._path_stat(self)
if not res:
return False
return res.text_mode in ("regular file", "regular empty file")
def is_symlink(self):
res = self.remote._path_stat(self)
if not res:
return False
return res.text_mode == "symbolic link"
def exists(self):
return self.remote._path_stat(self) is not None
def stat(self):
res = self.remote._path_stat(self)
if res is None:
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), "")
return res
def with_name(self, name):
return self.__class__(self.remote, self.dirname) / name
def with_suffix(self, suffix, depth=1):
if suffix and not suffix.startswith(".") or suffix == ".":
raise ValueError(f"Invalid suffix {suffix!r}")
name = self.name
depth = len(self.suffixes) if depth is None else min(depth, len(self.suffixes))
for _ in range(depth):
name, _ = name.rsplit(".", 1)
return self.__class__(self.remote, self.dirname) / (name + suffix)
def glob(self, pattern):
return self._glob(
pattern,
lambda pat: [
RemotePath(self.remote, m) for m in self.remote._path_glob(self, pat)
],
)
def delete(self):
if not self.exists():
return
self.remote._path_delete(self)
unlink = delete
def move(self, dst):
if isinstance(dst, RemotePath):
if dst.remote is not self.remote:
raise TypeError("dst points to a different remote machine")
elif not isinstance(dst, str):
raise TypeError(
f"dst must be a string or a RemotePath (to the same remote machine), got {dst!r}"
)
self.remote._path_move(self, dst)
def copy(self, dst, override=False):
if isinstance(dst, RemotePath):
if dst.remote is not self.remote:
raise TypeError("dst points to a different remote machine")
elif not isinstance(dst, str):
raise TypeError(
f"dst must be a string or a RemotePath (to the same remote machine), got {dst!r}"
)
if override:
if isinstance(dst, str):
dst = RemotePath(self.remote, dst)
dst.delete()
else:
if isinstance(dst, str):
dst = RemotePath(self.remote, dst)
if dst.exists():
raise TypeError("Override not specified and dst exists")
self.remote._path_copy(self, dst)
def mkdir(self, mode=None, parents=True, exist_ok=True):
if parents and exist_ok:
self.remote._path_mkdir(self, mode=mode, minus_p=True)
else:
if parents and len(self.parts) > 1:
self.remote._path_mkdir(self.parent, mode=mode, minus_p=True)
try:
self.remote._path_mkdir(self, mode=mode, minus_p=False)
except ProcessExecutionError as ex:
if "File exists" not in ex.stderr:
raise
if not exist_ok:
raise OSError(
errno.EEXIST, "File exists (on remote end)", str(self)
) from None
def read(self, encoding=None):
data = self.remote._path_read(self)
if encoding:
data = data.decode(encoding)
return data
def write(self, data, encoding=None):
if encoding:
data = data.encode(encoding)
self.remote._path_write(self, data)
def touch(self):
self.remote._path_touch(str(self))
def chown(self, owner=None, group=None, recursive=None):
self.remote._path_chown(
self, owner, group, self.is_dir() if recursive is None else recursive
)
def chmod(self, mode):
self.remote._path_chmod(mode, self)
def access(self, mode=0):
mode = self._access_mode_to_flags(mode)
res = self.remote._path_stat(self)
if res is None:
return False
mask = res.st_mode & 0x1FF
return ((mask >> 6) & mode) or ((mask >> 3) & mode)
def link(self, dst):
if isinstance(dst, RemotePath):
if dst.remote is not self.remote:
raise TypeError("dst points to a different remote machine")
elif not isinstance(dst, str):
raise TypeError(
f"dst must be a string or a RemotePath (to the same remote machine), got {dst!r}"
)
self.remote._path_link(self, dst, False)
def symlink(self, dst):
if isinstance(dst, RemotePath):
if dst.remote is not self.remote:
raise TypeError("dst points to a different remote machine")
elif not isinstance(dst, str):
raise TypeError(
"dst must be a string or a RemotePath (to the same remote machine), got {dst!r}"
)
self.remote._path_link(self, dst, True)
def open(self, mode="r", bufsize=-1, *, encoding=None):
"""
Opens this path as a file.
Only works for ParamikoMachine-associated paths for now.
"""
if encoding is not None:
raise NotImplementedError(
"encoding not supported for ParamikoMachine paths"
)
if hasattr(self.remote, "sftp") and hasattr(self.remote.sftp, "open"):
return self.remote.sftp.open(self, mode, bufsize)
raise NotImplementedError(
"RemotePath.open only works for ParamikoMachine-associated paths for now"
)
def as_uri(self, scheme="ssh"):
suffix = urllib.pathname2url(str(self))
return f"{scheme}://{self.remote._fqhost}{suffix}"
@property
def stem(self):
return self.name.rsplit(".")[0]
@property
def root(self):
return "/"
@property
def drive(self):
return ""
class RemoteWorkdir(RemotePath):
"""Remote working directory manipulator"""
def __new__(cls, remote):
self = super().__new__(cls, remote, remote._session.run("pwd")[1].strip())
return self
def __hash__(self):
raise TypeError("unhashable type")
def chdir(self, newdir):
"""Changes the current working directory to the given one"""
self.remote._session.run(f"cd {shquote(newdir)}")
if hasattr(self.remote, "_cwd"):
del self.remote._cwd
return self.__class__(self.remote)
def getpath(self):
"""Returns the current working directory as a
`remote path <plumbum.path.remote.RemotePath>` object"""
return RemotePath(self.remote, self)
@contextmanager
def __call__(self, newdir):
"""A context manager used to ``chdir`` into a directory and then ``chdir`` back to
the previous location; much like ``pushd``/``popd``.
:param newdir: The destination director (a string or a
:class:`RemotePath <plumbum.path.remote.RemotePath>`)
"""
prev = self._path
changed_dir = self.chdir(newdir)
try:
yield changed_dir
finally:
self.chdir(prev)

View File

@@ -0,0 +1,116 @@
import os
from plumbum.machines.local import local
from plumbum.path.base import Path
from plumbum.path.local import LocalPath
def delete(*paths):
"""Deletes the given paths. The arguments can be either strings,
:class:`local paths <plumbum.path.local.LocalPath>`,
:class:`remote paths <plumbum.path.remote.RemotePath>`, or iterables of such.
No error is raised if any of the paths does not exist (it is silently ignored)
"""
for p in paths:
if isinstance(p, Path):
p.delete()
elif isinstance(p, str):
local.path(p).delete()
elif hasattr(p, "__iter__"):
delete(*p)
else:
raise TypeError(f"Cannot delete {p!r}")
def _move(src, dst):
ret = copy(src, dst)
delete(src)
return ret
def move(src, dst):
"""Moves the source path onto the destination path; ``src`` and ``dst`` can be either
strings, :class:`LocalPaths <plumbum.path.local.LocalPath>` or
:class:`RemotePath <plumbum.path.remote.RemotePath>`; any combination of the three will
work.
.. versionadded:: 1.3
``src`` can also be a list of strings/paths, in which case ``dst`` must not exist or be a directory.
"""
if not isinstance(dst, Path):
dst = local.path(dst)
if isinstance(src, (tuple, list)):
if not dst.exists():
dst.mkdir()
elif not dst.is_dir():
raise ValueError(
f"When using multiple sources, dst {dst!r} must be a directory"
)
for src2 in src:
move(src2, dst)
return dst
if not isinstance(src, Path):
src = local.path(src)
if isinstance(src, LocalPath):
return src.move(dst) if isinstance(dst, LocalPath) else _move(src, dst)
if isinstance(dst, LocalPath):
return _move(src, dst)
if src.remote == dst.remote:
return src.move(dst)
return _move(src, dst)
def copy(src, dst):
"""
Copy (recursively) the source path onto the destination path; ``src`` and ``dst`` can be
either strings, :class:`LocalPaths <plumbum.path.local.LocalPath>` or
:class:`RemotePath <plumbum.path.remote.RemotePath>`; any combination of the three will
work.
.. versionadded:: 1.3
``src`` can also be a list of strings/paths, in which case ``dst`` must not exist or be a directory.
"""
if not isinstance(dst, Path):
dst = local.path(dst)
if isinstance(src, (tuple, list)):
if not dst.exists():
dst.mkdir()
elif not dst.is_dir():
raise ValueError(
f"When using multiple sources, dst {dst!r} must be a directory"
)
for src2 in src:
copy(src2, dst)
return dst
if not isinstance(src, Path):
src = local.path(src)
if isinstance(src, LocalPath):
if isinstance(dst, LocalPath):
return src.copy(dst)
dst.remote.upload(src, dst)
return dst
if isinstance(dst, LocalPath):
src.remote.download(src, dst)
return dst
if src.remote == dst.remote:
return src.copy(dst)
with local.tempdir() as tmp:
copy(src, tmp)
copy(tmp / src.name, dst)
return dst
def gui_open(filename):
"""This selects the proper gui open function. This can
also be achieved with webbrowser, but that is not supported."""
if hasattr(os, "startfile"):
os.startfile(filename)
else:
local.get("xdg-open", "open")(filename)

View File

@@ -0,0 +1,173 @@
import inspect
import os
from collections.abc import MutableMapping
NO_DEFAULT = object()
# must not inherit from AttributeError, so not to mess with python's attribute-lookup flow
class EnvironmentVariableError(KeyError):
pass
class TypedEnv(MutableMapping):
"""
This object can be used in 'exploratory' mode:
nv = TypedEnv()
print(nv.HOME)
It can also be used as a parser and validator of environment variables:
class MyEnv(TypedEnv):
username = TypedEnv.Str("USER", default='anonymous')
path = TypedEnv.CSV("PATH", separator=":")
tmp = TypedEnv.Str("TMP TEMP".split()) # support 'fallback' var-names
nv = MyEnv()
print(nv.username)
for p in nv.path:
print(p)
try:
print(p.tmp)
except EnvironmentVariableError:
print("TMP/TEMP is not defined")
else:
assert False
"""
__slots__ = ["_env", "_defined_keys"]
class _BaseVar:
def __init__(self, name, default=NO_DEFAULT):
self.names = tuple(name) if isinstance(name, (tuple, list)) else (name,)
self.name = self.names[0]
self.default = default
def convert(self, value): # pylint:disable=no-self-use
return value
def __get__(self, instance, owner):
if not instance:
return self
try:
return self.convert(instance._raw_get(*self.names))
except EnvironmentVariableError:
if self.default is NO_DEFAULT:
raise
return self.default
def __set__(self, instance, value):
instance[self.name] = value
class Str(_BaseVar):
pass
class Bool(_BaseVar):
"""
Converts 'yes|true|1|no|false|0' to the appropriate boolean value.
Case-insensitive. Throws a ``ValueError`` for any other value.
"""
def convert(self, value):
value = value.lower()
if value not in {"yes", "no", "true", "false", "1", "0"}:
raise ValueError(f"Unrecognized boolean value: {value!r}")
return value in {"yes", "true", "1"}
def __set__(self, instance, value):
instance[self.name] = "yes" if value else "no"
class Int(_BaseVar):
convert = staticmethod(int)
class Float(_BaseVar):
convert = staticmethod(float)
class CSV(_BaseVar):
"""
Comma-separated-strings get split using the ``separator`` (',' by default) into
a list of objects of type ``type`` (``str`` by default).
"""
def __init__(
self, name, default=NO_DEFAULT, type=str, separator=","
): # pylint:disable=redefined-builtin
super().__init__(name, default=default)
self.type = type
self.separator = separator
def __set__(self, instance, value):
instance[self.name] = self.separator.join(map(str, value))
def convert(self, value):
return [self.type(v.strip()) for v in value.split(self.separator)]
# =========
def __init__(self, env=None):
if env is None:
env = os.environ
self._env = env
self._defined_keys = {
k
for (k, v) in inspect.getmembers(self.__class__)
if isinstance(v, self._BaseVar)
}
def __iter__(self):
return iter(dir(self))
def __len__(self):
return len(self._env)
def __delitem__(self, name):
del self._env[name]
def __setitem__(self, name, value):
self._env[name] = str(value)
def _raw_get(self, *key_names):
for key in key_names:
value = self._env.get(key, NO_DEFAULT)
if value is not NO_DEFAULT:
return value
raise EnvironmentVariableError(key_names[0])
def __contains__(self, key):
try:
self._raw_get(key)
except EnvironmentVariableError:
return False
else:
return True
def __getattr__(self, name):
# if we're here then there was no descriptor defined
try:
return self._raw_get(name)
except EnvironmentVariableError:
raise AttributeError(
f"{self.__class__} has no attribute {name!r}"
) from None
def __getitem__(self, key):
return getattr(self, key) # delegate through the descriptors
def get(self, key, default=None):
try:
return self[key]
except EnvironmentVariableError:
return default
def __dir__(self):
if self._defined_keys:
# return only defined
return sorted(self._defined_keys)
# return whatever is in the environment (for convenience)
members = set(self._env.keys())
members.update(dir(self.__class__))
return sorted(members)

View File

@@ -0,0 +1,4 @@
# file generated by setuptools_scm
# don't change, don't track in version control
__version__ = version = '1.8.1'
__version_tuple__ = version_tuple = (1, 8, 1)