Add pexpect-2.4 (a pure Python module for controlling and automating other programs) to the test directory.

http://pypi.python.org/pypi/pexpect/

llvm-svn: 127484
This commit is contained in:
Johnny Chen 2011-03-11 20:13:06 +00:00
parent 0b5119315b
commit b7cfba4cb1
40 changed files with 9758 additions and 2 deletions

View File

@ -496,7 +496,10 @@ def parseOptionsAndInitTestdirs():
def setupSysPath():
"""Add LLDB.framework/Resources/Python to the search paths for modules."""
"""
Add LLDB.framework/Resources/Python to the search paths for modules.
As a side effect, we also discover the 'lldb' executable and export it here.
"""
global rdir
global testdirs
@ -523,10 +526,12 @@ def setupSysPath():
else:
os.environ["LLDB_TEST"] = scriptPath
pluginPath = os.path.join(scriptPath, 'plugins')
pexpectPath = os.path.join(scriptPath, 'pexpect-2.4')
# Append script dir and plugin dir to the sys.path.
# Append script dir, plugin dir, and pexpect dir to the sys.path.
sys.path.append(scriptPath)
sys.path.append(pluginPath)
sys.path.append(pexpectPath)
# This is our base name component.
base = os.path.abspath(os.path.join(scriptPath, os.pardir))

View File

@ -0,0 +1,334 @@
"""This implements an ANSI terminal emulator as a subclass of screen.
$Id: ANSI.py 491 2007-12-16 20:04:57Z noah $
"""
# references:
# http://www.retards.org/terminals/vt102.html
# http://vt100.net/docs/vt102-ug/contents.html
# http://vt100.net/docs/vt220-rm/
# http://www.termsys.demon.co.uk/vtansi.htm
import screen
import FSM
import copy
import string
def Emit (fsm):
screen = fsm.memory[0]
screen.write_ch(fsm.input_symbol)
def StartNumber (fsm):
fsm.memory.append (fsm.input_symbol)
def BuildNumber (fsm):
ns = fsm.memory.pop()
ns = ns + fsm.input_symbol
fsm.memory.append (ns)
def DoBackOne (fsm):
screen = fsm.memory[0]
screen.cursor_back ()
def DoBack (fsm):
count = int(fsm.memory.pop())
screen = fsm.memory[0]
screen.cursor_back (count)
def DoDownOne (fsm):
screen = fsm.memory[0]
screen.cursor_down ()
def DoDown (fsm):
count = int(fsm.memory.pop())
screen = fsm.memory[0]
screen.cursor_down (count)
def DoForwardOne (fsm):
screen = fsm.memory[0]
screen.cursor_forward ()
def DoForward (fsm):
count = int(fsm.memory.pop())
screen = fsm.memory[0]
screen.cursor_forward (count)
def DoUpReverse (fsm):
screen = fsm.memory[0]
screen.cursor_up_reverse()
def DoUpOne (fsm):
screen = fsm.memory[0]
screen.cursor_up ()
def DoUp (fsm):
count = int(fsm.memory.pop())
screen = fsm.memory[0]
screen.cursor_up (count)
def DoHome (fsm):
c = int(fsm.memory.pop())
r = int(fsm.memory.pop())
screen = fsm.memory[0]
screen.cursor_home (r,c)
def DoHomeOrigin (fsm):
c = 1
r = 1
screen = fsm.memory[0]
screen.cursor_home (r,c)
def DoEraseDown (fsm):
screen = fsm.memory[0]
screen.erase_down()
def DoErase (fsm):
arg = int(fsm.memory.pop())
screen = fsm.memory[0]
if arg == 0:
screen.erase_down()
elif arg == 1:
screen.erase_up()
elif arg == 2:
screen.erase_screen()
def DoEraseEndOfLine (fsm):
screen = fsm.memory[0]
screen.erase_end_of_line()
def DoEraseLine (fsm):
screen = fsm.memory[0]
if arg == 0:
screen.end_of_line()
elif arg == 1:
screen.start_of_line()
elif arg == 2:
screen.erase_line()
def DoEnableScroll (fsm):
screen = fsm.memory[0]
screen.scroll_screen()
def DoCursorSave (fsm):
screen = fsm.memory[0]
screen.cursor_save_attrs()
def DoCursorRestore (fsm):
screen = fsm.memory[0]
screen.cursor_restore_attrs()
def DoScrollRegion (fsm):
screen = fsm.memory[0]
r2 = int(fsm.memory.pop())
r1 = int(fsm.memory.pop())
screen.scroll_screen_rows (r1,r2)
def DoMode (fsm):
screen = fsm.memory[0]
mode = fsm.memory.pop() # Should be 4
# screen.setReplaceMode ()
def Log (fsm):
screen = fsm.memory[0]
fsm.memory = [screen]
fout = open ('log', 'a')
fout.write (fsm.input_symbol + ',' + fsm.current_state + '\n')
fout.close()
class term (screen.screen):
"""This is a placeholder.
In theory I might want to add other terminal types.
"""
def __init__ (self, r=24, c=80):
screen.screen.__init__(self, r,c)
class ANSI (term):
"""This class encapsulates a generic terminal. It filters a stream and
maintains the state of a screen object. """
def __init__ (self, r=24,c=80):
term.__init__(self,r,c)
#self.screen = screen (24,80)
self.state = FSM.FSM ('INIT',[self])
self.state.set_default_transition (Log, 'INIT')
self.state.add_transition_any ('INIT', Emit, 'INIT')
self.state.add_transition ('\x1b', 'INIT', None, 'ESC')
self.state.add_transition_any ('ESC', Log, 'INIT')
self.state.add_transition ('(', 'ESC', None, 'G0SCS')
self.state.add_transition (')', 'ESC', None, 'G1SCS')
self.state.add_transition_list ('AB012', 'G0SCS', None, 'INIT')
self.state.add_transition_list ('AB012', 'G1SCS', None, 'INIT')
self.state.add_transition ('7', 'ESC', DoCursorSave, 'INIT')
self.state.add_transition ('8', 'ESC', DoCursorRestore, 'INIT')
self.state.add_transition ('M', 'ESC', DoUpReverse, 'INIT')
self.state.add_transition ('>', 'ESC', DoUpReverse, 'INIT')
self.state.add_transition ('<', 'ESC', DoUpReverse, 'INIT')
self.state.add_transition ('=', 'ESC', None, 'INIT') # Selects application keypad.
self.state.add_transition ('#', 'ESC', None, 'GRAPHICS_POUND')
self.state.add_transition_any ('GRAPHICS_POUND', None, 'INIT')
self.state.add_transition ('[', 'ESC', None, 'ELB')
# ELB means Escape Left Bracket. That is ^[[
self.state.add_transition ('H', 'ELB', DoHomeOrigin, 'INIT')
self.state.add_transition ('D', 'ELB', DoBackOne, 'INIT')
self.state.add_transition ('B', 'ELB', DoDownOne, 'INIT')
self.state.add_transition ('C', 'ELB', DoForwardOne, 'INIT')
self.state.add_transition ('A', 'ELB', DoUpOne, 'INIT')
self.state.add_transition ('J', 'ELB', DoEraseDown, 'INIT')
self.state.add_transition ('K', 'ELB', DoEraseEndOfLine, 'INIT')
self.state.add_transition ('r', 'ELB', DoEnableScroll, 'INIT')
self.state.add_transition ('m', 'ELB', None, 'INIT')
self.state.add_transition ('?', 'ELB', None, 'MODECRAP')
self.state.add_transition_list (string.digits, 'ELB', StartNumber, 'NUMBER_1')
self.state.add_transition_list (string.digits, 'NUMBER_1', BuildNumber, 'NUMBER_1')
self.state.add_transition ('D', 'NUMBER_1', DoBack, 'INIT')
self.state.add_transition ('B', 'NUMBER_1', DoDown, 'INIT')
self.state.add_transition ('C', 'NUMBER_1', DoForward, 'INIT')
self.state.add_transition ('A', 'NUMBER_1', DoUp, 'INIT')
self.state.add_transition ('J', 'NUMBER_1', DoErase, 'INIT')
self.state.add_transition ('K', 'NUMBER_1', DoEraseLine, 'INIT')
self.state.add_transition ('l', 'NUMBER_1', DoMode, 'INIT')
### It gets worse... the 'm' code can have infinite number of
### number;number;number before it. I've never seen more than two,
### but the specs say it's allowed. crap!
self.state.add_transition ('m', 'NUMBER_1', None, 'INIT')
### LED control. Same problem as 'm' code.
self.state.add_transition ('q', 'NUMBER_1', None, 'INIT')
# \E[?47h appears to be "switch to alternate screen"
# \E[?47l restores alternate screen... I think.
self.state.add_transition_list (string.digits, 'MODECRAP', StartNumber, 'MODECRAP_NUM')
self.state.add_transition_list (string.digits, 'MODECRAP_NUM', BuildNumber, 'MODECRAP_NUM')
self.state.add_transition ('l', 'MODECRAP_NUM', None, 'INIT')
self.state.add_transition ('h', 'MODECRAP_NUM', None, 'INIT')
#RM Reset Mode Esc [ Ps l none
self.state.add_transition (';', 'NUMBER_1', None, 'SEMICOLON')
self.state.add_transition_any ('SEMICOLON', Log, 'INIT')
self.state.add_transition_list (string.digits, 'SEMICOLON', StartNumber, 'NUMBER_2')
self.state.add_transition_list (string.digits, 'NUMBER_2', BuildNumber, 'NUMBER_2')
self.state.add_transition_any ('NUMBER_2', Log, 'INIT')
self.state.add_transition ('H', 'NUMBER_2', DoHome, 'INIT')
self.state.add_transition ('f', 'NUMBER_2', DoHome, 'INIT')
self.state.add_transition ('r', 'NUMBER_2', DoScrollRegion, 'INIT')
### It gets worse... the 'm' code can have infinite number of
### number;number;number before it. I've never seen more than two,
### but the specs say it's allowed. crap!
self.state.add_transition ('m', 'NUMBER_2', None, 'INIT')
### LED control. Same problem as 'm' code.
self.state.add_transition ('q', 'NUMBER_2', None, 'INIT')
def process (self, c):
self.state.process(c)
def process_list (self, l):
self.write(l)
def write (self, s):
for c in s:
self.process(c)
def flush (self):
pass
def write_ch (self, ch):
"""This puts a character at the current cursor position. cursor
position if moved forward with wrap-around, but no scrolling is done if
the cursor hits the lower-right corner of the screen. """
#\r and \n both produce a call to crlf().
ch = ch[0]
if ch == '\r':
# self.crlf()
return
if ch == '\n':
self.crlf()
return
if ch == chr(screen.BS):
self.cursor_back()
self.put_abs(self.cur_r, self.cur_c, ' ')
return
if ch not in string.printable:
fout = open ('log', 'a')
fout.write ('Nonprint: ' + str(ord(ch)) + '\n')
fout.close()
return
self.put_abs(self.cur_r, self.cur_c, ch)
old_r = self.cur_r
old_c = self.cur_c
self.cursor_forward()
if old_c == self.cur_c:
self.cursor_down()
if old_r != self.cur_r:
self.cursor_home (self.cur_r, 1)
else:
self.scroll_up ()
self.cursor_home (self.cur_r, 1)
self.erase_line()
# def test (self):
#
# import sys
# write_text = 'I\'ve got a ferret sticking up my nose.\n' + \
# '(He\'s got a ferret sticking up his nose.)\n' + \
# 'How it got there I can\'t tell\n' + \
# 'But now it\'s there it hurts like hell\n' + \
# 'And what is more it radically affects my sense of smell.\n' + \
# '(His sense of smell.)\n' + \
# 'I can see a bare-bottomed mandril.\n' + \
# '(Slyly eyeing his other nostril.)\n' + \
# 'If it jumps inside there too I really don\'t know what to do\n' + \
# 'I\'ll be the proud posessor of a kind of nasal zoo.\n' + \
# '(A nasal zoo.)\n' + \
# 'I\'ve got a ferret sticking up my nose.\n' + \
# '(And what is worst of all it constantly explodes.)\n' + \
# '"Ferrets don\'t explode," you say\n' + \
# 'But it happened nine times yesterday\n' + \
# 'And I should know for each time I was standing in the way.\n' + \
# 'I\'ve got a ferret sticking up my nose.\n' + \
# '(He\'s got a ferret sticking up his nose.)\n' + \
# 'How it got there I can\'t tell\n' + \
# 'But now it\'s there it hurts like hell\n' + \
# 'And what is more it radically affects my sense of smell.\n' + \
# '(His sense of smell.)'
# self.fill('.')
# self.cursor_home()
# for c in write_text:
# self.write_ch (c)
# print str(self)
#
#if __name__ == '__main__':
# t = ANSI(6,65)
# t.test()

View File

@ -0,0 +1,331 @@
#!/usr/bin/env python
"""This module implements a Finite State Machine (FSM). In addition to state
this FSM also maintains a user defined "memory". So this FSM can be used as a
Push-down Automata (PDA) since a PDA is a FSM + memory.
The following describes how the FSM works, but you will probably also need to
see the example function to understand how the FSM is used in practice.
You define an FSM by building tables of transitions. For a given input symbol
the process() method uses these tables to decide what action to call and what
the next state will be. The FSM has a table of transitions that associate:
(input_symbol, current_state) --> (action, next_state)
Where "action" is a function you define. The symbols and states can be any
objects. You use the add_transition() and add_transition_list() methods to add
to the transition table. The FSM also has a table of transitions that
associate:
(current_state) --> (action, next_state)
You use the add_transition_any() method to add to this transition table. The
FSM also has one default transition that is not associated with any specific
input_symbol or state. You use the set_default_transition() method to set the
default transition.
When an action function is called it is passed a reference to the FSM. The
action function may then access attributes of the FSM such as input_symbol,
current_state, or "memory". The "memory" attribute can be any object that you
want to pass along to the action functions. It is not used by the FSM itself.
For parsing you would typically pass a list to be used as a stack.
The processing sequence is as follows. The process() method is given an
input_symbol to process. The FSM will search the table of transitions that
associate:
(input_symbol, current_state) --> (action, next_state)
If the pair (input_symbol, current_state) is found then process() will call the
associated action function and then set the current state to the next_state.
If the FSM cannot find a match for (input_symbol, current_state) it will then
search the table of transitions that associate:
(current_state) --> (action, next_state)
If the current_state is found then the process() method will call the
associated action function and then set the current state to the next_state.
Notice that this table lacks an input_symbol. It lets you define transitions
for a current_state and ANY input_symbol. Hence, it is called the "any" table.
Remember, it is always checked after first searching the table for a specific
(input_symbol, current_state).
For the case where the FSM did not match either of the previous two cases the
FSM will try to use the default transition. If the default transition is
defined then the process() method will call the associated action function and
then set the current state to the next_state. This lets you define a default
transition as a catch-all case. You can think of it as an exception handler.
There can be only one default transition.
Finally, if none of the previous cases are defined for an input_symbol and
current_state then the FSM will raise an exception. This may be desirable, but
you can always prevent this just by defining a default transition.
Noah Spurrier 20020822
"""
class ExceptionFSM(Exception):
"""This is the FSM Exception class."""
def __init__(self, value):
self.value = value
def __str__(self):
return `self.value`
class FSM:
"""This is a Finite State Machine (FSM).
"""
def __init__(self, initial_state, memory=None):
"""This creates the FSM. You set the initial state here. The "memory"
attribute is any object that you want to pass along to the action
functions. It is not used by the FSM. For parsing you would typically
pass a list to be used as a stack. """
# Map (input_symbol, current_state) --> (action, next_state).
self.state_transitions = {}
# Map (current_state) --> (action, next_state).
self.state_transitions_any = {}
self.default_transition = None
self.input_symbol = None
self.initial_state = initial_state
self.current_state = self.initial_state
self.next_state = None
self.action = None
self.memory = memory
def reset (self):
"""This sets the current_state to the initial_state and sets
input_symbol to None. The initial state was set by the constructor
__init__(). """
self.current_state = self.initial_state
self.input_symbol = None
def add_transition (self, input_symbol, state, action=None, next_state=None):
"""This adds a transition that associates:
(input_symbol, current_state) --> (action, next_state)
The action may be set to None in which case the process() method will
ignore the action and only set the next_state. The next_state may be
set to None in which case the current state will be unchanged.
You can also set transitions for a list of symbols by using
add_transition_list(). """
if next_state is None:
next_state = state
self.state_transitions[(input_symbol, state)] = (action, next_state)
def add_transition_list (self, list_input_symbols, state, action=None, next_state=None):
"""This adds the same transition for a list of input symbols.
You can pass a list or a string. Note that it is handy to use
string.digits, string.whitespace, string.letters, etc. to add
transitions that match character classes.
The action may be set to None in which case the process() method will
ignore the action and only set the next_state. The next_state may be
set to None in which case the current state will be unchanged. """
if next_state is None:
next_state = state
for input_symbol in list_input_symbols:
self.add_transition (input_symbol, state, action, next_state)
def add_transition_any (self, state, action=None, next_state=None):
"""This adds a transition that associates:
(current_state) --> (action, next_state)
That is, any input symbol will match the current state.
The process() method checks the "any" state associations after it first
checks for an exact match of (input_symbol, current_state).
The action may be set to None in which case the process() method will
ignore the action and only set the next_state. The next_state may be
set to None in which case the current state will be unchanged. """
if next_state is None:
next_state = state
self.state_transitions_any [state] = (action, next_state)
def set_default_transition (self, action, next_state):
"""This sets the default transition. This defines an action and
next_state if the FSM cannot find the input symbol and the current
state in the transition list and if the FSM cannot find the
current_state in the transition_any list. This is useful as a final
fall-through state for catching errors and undefined states.
The default transition can be removed by setting the attribute
default_transition to None. """
self.default_transition = (action, next_state)
def get_transition (self, input_symbol, state):
"""This returns (action, next state) given an input_symbol and state.
This does not modify the FSM state, so calling this method has no side
effects. Normally you do not call this method directly. It is called by
process().
The sequence of steps to check for a defined transition goes from the
most specific to the least specific.
1. Check state_transitions[] that match exactly the tuple,
(input_symbol, state)
2. Check state_transitions_any[] that match (state)
In other words, match a specific state and ANY input_symbol.
3. Check if the default_transition is defined.
This catches any input_symbol and any state.
This is a handler for errors, undefined states, or defaults.
4. No transition was defined. If we get here then raise an exception.
"""
if self.state_transitions.has_key((input_symbol, state)):
return self.state_transitions[(input_symbol, state)]
elif self.state_transitions_any.has_key (state):
return self.state_transitions_any[state]
elif self.default_transition is not None:
return self.default_transition
else:
raise ExceptionFSM ('Transition is undefined: (%s, %s).' %
(str(input_symbol), str(state)) )
def process (self, input_symbol):
"""This is the main method that you call to process input. This may
cause the FSM to change state and call an action. This method calls
get_transition() to find the action and next_state associated with the
input_symbol and current_state. If the action is None then the action
is not called and only the current state is changed. This method
processes one complete input symbol. You can process a list of symbols
(or a string) by calling process_list(). """
self.input_symbol = input_symbol
(self.action, self.next_state) = self.get_transition (self.input_symbol, self.current_state)
if self.action is not None:
self.action (self)
self.current_state = self.next_state
self.next_state = None
def process_list (self, input_symbols):
"""This takes a list and sends each element to process(). The list may
be a string or any iterable object. """
for s in input_symbols:
self.process (s)
##############################################################################
# The following is an example that demonstrates the use of the FSM class to
# process an RPN expression. Run this module from the command line. You will
# get a prompt > for input. Enter an RPN Expression. Numbers may be integers.
# Operators are * / + - Use the = sign to evaluate and print the expression.
# For example:
#
# 167 3 2 2 * * * 1 - =
#
# will print:
#
# 2003
##############################################################################
import sys, os, traceback, optparse, time, string
#
# These define the actions.
# Note that "memory" is a list being used as a stack.
#
def BeginBuildNumber (fsm):
fsm.memory.append (fsm.input_symbol)
def BuildNumber (fsm):
s = fsm.memory.pop ()
s = s + fsm.input_symbol
fsm.memory.append (s)
def EndBuildNumber (fsm):
s = fsm.memory.pop ()
fsm.memory.append (int(s))
def DoOperator (fsm):
ar = fsm.memory.pop()
al = fsm.memory.pop()
if fsm.input_symbol == '+':
fsm.memory.append (al + ar)
elif fsm.input_symbol == '-':
fsm.memory.append (al - ar)
elif fsm.input_symbol == '*':
fsm.memory.append (al * ar)
elif fsm.input_symbol == '/':
fsm.memory.append (al / ar)
def DoEqual (fsm):
print str(fsm.memory.pop())
def Error (fsm):
print 'That does not compute.'
print str(fsm.input_symbol)
def main():
"""This is where the example starts and the FSM state transitions are
defined. Note that states are strings (such as 'INIT'). This is not
necessary, but it makes the example easier to read. """
f = FSM ('INIT', []) # "memory" will be used as a stack.
f.set_default_transition (Error, 'INIT')
f.add_transition_any ('INIT', None, 'INIT')
f.add_transition ('=', 'INIT', DoEqual, 'INIT')
f.add_transition_list (string.digits, 'INIT', BeginBuildNumber, 'BUILDING_NUMBER')
f.add_transition_list (string.digits, 'BUILDING_NUMBER', BuildNumber, 'BUILDING_NUMBER')
f.add_transition_list (string.whitespace, 'BUILDING_NUMBER', EndBuildNumber, 'INIT')
f.add_transition_list ('+-*/', 'INIT', DoOperator, 'INIT')
print
print 'Enter an RPN Expression.'
print 'Numbers may be integers. Operators are * / + -'
print 'Use the = sign to evaluate and print the expression.'
print 'For example: '
print ' 167 3 2 2 * * * 1 - ='
inputstr = raw_input ('> ')
f.process_list(inputstr)
if __name__ == '__main__':
try:
start_time = time.time()
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), usage=globals()['__doc__'], version='$Id: FSM.py 490 2007-12-07 15:46:24Z noah $')
parser.add_option ('-v', '--verbose', action='store_true', default=False, help='verbose output')
(options, args) = parser.parse_args()
if options.verbose: print time.asctime()
main()
if options.verbose: print time.asctime()
if options.verbose: print 'TOTAL TIME IN MINUTES:',
if options.verbose: print (time.time() - start_time) / 60.0
sys.exit(0)
except KeyboardInterrupt, e: # Ctrl-C
raise e
except SystemExit, e: # sys.exit()
raise e
except Exception, e:
print 'ERROR, UNEXPECTED EXCEPTION'
print str(e)
traceback.print_exc()
os._exit(1)

View File

@ -0,0 +1,31 @@
Installation
------------
This is a standard Python Distutil distribution. To install simply run:
python setup.py install
This makes Pexpect available to any script on the machine. You need
root access to install it this way. If you do not have root access or
if you do not wish to install Pexpect so that is available to any script
then you can just copy the pexpect.py file to same directory as your script.
Trouble on Debian and Ubuntu
----------------------------
For some stupid reason Debian Linux does not include the distutils module
in the standard 'python' package. Instead, the distutils module is packaged
separately in the 'python-dev' package. So to add distutils back
into Python, simply use aptitude or apt-get to install 'python-dev'.
As root, run this command:
apt-get install python-dev
Why they do this is mysterious because:
- It breaks the Python model of "batteries included".
'distutils' isn't an extra or optional module --
it's parts of the Standard Python Library.
- The Debian 'python-dev' package is a microscopic 50K installed.
So what are they saving?
- Distutils is not only interesting to developers. Many non-development
oriented Python packages use 'distutils' to install applications.
- As far as I can tell, the package maintainers must go through
more trouble to remove 'distutils' from the standard Python
distribution than it would take just to leave it in.

View File

@ -0,0 +1,21 @@
Free, open source, and all that good stuff.
Pexpect Copyright (c) 2008 Noah Spurrier
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@ -0,0 +1,10 @@
Metadata-Version: 1.0
Name: pexpect
Version: 2.4
Summary: Pexpect is a pure Python Expect. It allows easy control of other applications.
Home-page: http://pexpect.sourceforge.net/
Author: Noah Spurrier
Author-email: noah@noah.org
License: MIT license
Description: UNKNOWN
Platform: UNIX

View File

@ -0,0 +1,45 @@
Pexpect is a Pure Python Expect-like module
Pexpect makes Python a better tool for controlling other applications.
Pexpect is a pure Python module for spawning child applications; controlling
them; and responding to expected patterns in their output. Pexpect works like
Don Libes' Expect. Pexpect allows your script to spawn a child application and
control it as if a human were typing commands.
Pexpect can be used for automating interactive applications such as ssh, ftp,
passwd, telnet, etc. It can be used to a automate setup scripts for
duplicating software package installations on different servers. It can be
used for automated software testing. Pexpect is in the spirit of Don Libes'
Expect, but Pexpect is pure Python. Unlike other Expect-like modules for
Python, Pexpect does not require TCL or Expect nor does it require C
extensions to be compiled. It should work on any platform that supports the
standard Python pty module. The Pexpect interface was designed to be easy to use.
If you want to work with the development version of the source code then please
read the DEVELOPERS document in the root of the source code tree.
Free, open source, and all that good stuff.
Pexpect Copyright (c) 2008 Noah Spurrier
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
Noah Spurrier
http://pexpect.sourceforge.net/

View File

@ -0,0 +1,103 @@
body {
margin:0px;
padding:0px;
font-family:verdana, arial, helvetica, sans-serif;
color:#333;
background-color:white;
}
pre {
background: #eeeeee;
border: 1px solid #888888;
color: black;
padding: 1em;
white-space: pre;
}
h1 {
margin:5px 0px 5px 0px;
padding:0px;
font-size:20px;
line-height:28px;
font-weight:900;
color:#44f;
}
h2 {
margin:5px 0px 5px 0px;
padding:0px;
font-size:17px;
line-height:28px;
font-weight:900;
color:#226;
}
h3 {
margin:5px 0px 5px 0px;
padding:0px;
font-size:15px;
line-height:28px;
font-weight:900;
}
p
{
margin:0px 0px 16px 0px;
font:11px/20px verdana, arial, helvetica, sans-serif;
padding:0px;
}
table
{
font-size: 10pt;
color: #000000;
}
td{border:1px solid #999;}
table.pymenu {color: #000000; background-color: #99ccff}
th.pymenu {color: #ffffff; background-color: #003366}
.code
{
font-family: "Lucida Console", monospace; font-weight: bold;
color: #007700; background-color: #eeeeee
}
#Content>p {margin:0px;}
#Content>p+p {text-indent:30px;}
a {
text-decoration:none;
font-weight:600;
font-family:verdana, arial, helvetica, sans-serif;
color: #900;
}
//a:link {color:#09c;}
//a x:visited {color:#07a;}
a:hover {background-color:#ee0;}
#Header {
margin:10px 0px 10px 0px;
padding:10px 0px 10px 20px;
/* For IE5/Win's benefit height = [correct height] + [top padding] + [top and bottom border widths] */
height:33px; /* 14px + 17px + 2px = 33px */
border-style:solid;
border-color:black;
border-width:1px 0px; /* top and bottom borders: 1px; left and right borders: 0px */
line-height:33px;
background-color:#eee;
height:66px; /* the correct height */
}
#Content {
margin:0px 210px 50px 10px;
padding:10px;
}
#Menu {
position:absolute;
top:100px;
right:20px;
width:172px;
padding:10px;
background-color:#eee;
border:1px solid #999; // dashed #999;
line-height:17px;
width:150px;
font-size:11px;
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 322 B

View File

@ -0,0 +1,135 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<title>Pexpect - Examples</title>
<link rel="stylesheet" href="clean.css" type="text/css">
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<meta name="Author" content="Noah Spurrier">
<meta name="Keywords"
content="pexpect, Noah Spurrier, Python, Libes, TCL, Expect, pipe, popen, pyExpect, expectpy, expect-like, expect-alike, expect like">
<meta name="Description" content="Examples for using Pexpect.">
</head>
<body bgcolor="#ffffff" text="#000000">
<div id="Header">
<h1>Pexpect Examples</h1>
</div>
<div id="Content">
<p><span class="code">hive.py</span></p>
<p><blockquote>
This script creates SSH connections to a list of hosts that
you provide. Then you are given a command line prompt. Each
shell command that you enter is sent to all the hosts. The
response from each host is collected and printed. For example,
you could connect to a dozen different machines and reboot
them all at once.
</p></blockquote>
<p><span class="code">script.py</span></p>
<p><blockquote>
This implements a command similar to the classic BSD
"script" command.
This will start a subshell and log all input and
output to a file.
This demonstrates the interact() method of Pexpect.
</p></blockquote>
<p><span class="code">fix_cvs_files.py</span></p>
<p><blockquote>
This is for cleaning up binary files improperly
added to CVS.
This script scans the given path to find binary
files;
checks with CVS to see if the sticky options are set
to -kb;
finally if sticky options are not -kb then uses 'cvs
admin' to
set the -kb option.
</p></blockquote>
<p><span class="code">ftp.py</span></p>
<p><blockquote>
This demonstrates an FTP "bookmark".
This connects to an ftp site; does a few ftp stuff;
and then gives the user
interactive control over the session. In this case
the "bookmark" is to a
directory on the OpenBSD ftp server. It puts you in
the i386 packages
directory. You can easily modify this for other
sites.
This demonstrates the interact() method of Pexpect.
</p></blockquote>
<p><span class="code">monitor.py</span></p>
<p><blockquote>
This runs a sequence of commands on a remote host
using SSH.
It runs a simple system checks such as uptime and
free to monitor
the state of the remote host.
</p></blockquote>
<p><span class="code">passmass.py</span></p>
<p><blockquote>
This will login to each given server and change the
password of the
given user. This demonstrates scripting logins and
passwords.
</p></blockquote>
<p><span class="code">python.py</span></p>
<p><blockquote>
This starts the python interpreter and prints the
greeting message backwards.
It then gives the user iteractive control of Python.
It's pretty useless!
</p></blockquote>
<p><span class="code">rippy.py</span></p>
<p><blockquote>
This is a wizard for mencoder. It greatly simplifies
the process of
ripping a DVD to Divx (mpeg4) format. It can
transcode from any
video file to another. It has options for resampling
the audio stream;
removing interlace artifacts, fitting to a target
file size, etc.
There are lots of options, but the process is simple
and easy to use.
</p></blockquote>
<p><span class="code">sshls.py</span></p>
<p><blockquote>
This lists a directory on a remote machine.
</p></blockquote>
<p><span class="code">ssh_tunnel.py</span></p>
<p><blockquote>
This starts an SSH tunnel to a remote machine. It
monitors the connection
and restarts the tunnel if it goes down.
</p></blockquote>
<p><span class="code">uptime.py</span></p>
<p><blockquote>
This will run the uptime command and parse the
output into variables.
This demonstrates using a single regular expression
to match the output
of a command and capturing different variable in
match groups.
The grouping regular expression handles a wide variety of different
uptime formats.
</blockquote>
<p>
<a href="http://sourceforge.net/projects/pexpect/"
title="The Pexpect project page on SourceForge.net"> <img
src="http://sourceforge.net/sflogo.php?group_id=59762&amp;type=5"
alt="The Pexpect project page on SourceForge.net" border="0"
height="31" width="105"> </a>
</p>
</div>
</body>
</html>

View File

@ -0,0 +1,868 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<title>Pexpect - a Pure Python Expect-like module</title>
<link rel="stylesheet" href="clean.css" type="text/css">
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<meta name="Author" content="Noah Spurrier">
<meta name="Keywords"
content="pexpect, Noah Spurrier, pypect, Python, Libes, TCL, Expect, pipe, popen, pyExpect, expectpy, expect-like, expect-alike, expect like">
<meta name="Description"
content="Pexpect is a pure Python Expect-like module. Pexpect makes Python a better tool for controlling other applications.">
</head>
<body bgcolor="#ffffff" text="#000000">
<div id="Header">
<h1>Pexpect version 2.4<br>
a Pure Python Expect-like module
</h1>
</div>
<div id="Content">
<p>Pexpect makes Python a better tool for controlling other
applications.</p>
<p>Pexpect is a pure Python module for spawning child applications;
controlling them; and responding to expected patterns in their output.
Pexpect works like Don Libes' Expect. Pexpect allows your script to
spawn a child application and control it as if a human were typing
commands.</p>
<p>Pexpect can be used for automating interactive applications such as
ssh, ftp, passwd, telnet, etc. It can be used to a automate setup
scripts for duplicating software package installations on different
servers. It can be used for automated software testing. Pexpect is in
the spirit of Don Libes' Expect, but Pexpect is pure Python. Unlike
other Expect-like modules for Python, Pexpect does not require TCL or
Expect nor does it require C extensions to be compiled. It should work
on any platform that supports the standard Python pty module. The
Pexpect interface was designed to be easy to use.</p>
<table border="0">
<tbody>
<tr>
<td align="right" valign="top">Send questions to:</td>
<td align="left"><a href="http://www.noah.org/email/"><img
src="email.png" alt="Click to send email." border="0" height="16"
width="100"></a></td>
</tr>
</tbody>
</table>
<hr noshade="noshade" size="1">
<h1><a name="license"></a>License: MIT style</h1>
<p>
Free, open source, and all that good stuff.<br>
<br>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:<br>
<br>
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.<br>
<br>
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.<br>
<br>
Pexpect Copyright (c) 2008 Noah Spurrier<br>
http://pexpect.sourceforge.net/
</p>
<hr noshade="noshade" size="1">
<h1><a name="download"></a><a
href="http://sourceforge.net/project/showfiles.php?group_id=59762">Download</a></h1>
<p>Download the <a
href="http://sourceforge.net/project/showfiles.php?group_id=59762">
current version here</a> from the SourceForge site. Grab the current Pexpect tarball.
</p>
<h2>Installing Pexpect</h2>
<p>The Pexpect tarball is a standard Python Distutil distribution.</p>
<ol>
<li>download <span class="code">pexpect-2.4.tar.gz</span></li>
<li><span class="code">tar zxf pexpect-2.4.tar.gz</span></li>
<li><span class="code">cd pexpect-2.4</span></li>
<li><span class="code">python setup.py install</span> <i>do this as root</i></li>
</ol>
<h2>Examples</h2>
<p>
Under the <span class="code">pexpect-2.4</span> directory you should find
the <span class="code">examples</span> directory.
This is the best way to learn to use Pexpect.
See the descriptions of <a href="examples.html">Pexpect Examples</a>.
</p>
<h2><a name="doc"></a>API Documentation</h2>
<p>
<blockquote>
<a href="pexpect.html">pexpect</a> This is the main module that you want.<br>
<a href="pxssh.html">pxssh</a> Pexpect SSH is an extension of 'pexpect.spawn' that specializes in SSH.<br>
</blockquote>
the following are experimental extensions to Pexpect<br>
<blockquote>
<a href="fdpexpect.html">fdpexpect</a> fdpexpect extension of 'pexpect.spawn' that uses an open file descriptor.<br>
<a href="screen.html">SCREEN</a> This represents a virtual 'screen'.<br>
<a href="ANSI.html">ANSI</a> This parses ANSI/VT-100 terminal escape codes.<br>
<a href="FSM.html">FSM</a> This is a finite state machine used by ANSI.<br>
</blockquote>
</p>
<hr noshade="noshade" size="1">
<h1><a name="status"></a>Project Status</h1>
<p>Automated pyunit tests reach over 80%
code coverage on pexpect.py. I regularly test on Linux and BSD
platforms. I try to test on Solaris and Irix.
</p>
<hr noshade="noshade" size="1">
<h1><a name="requirements"></a>Requirements for use of Pexpect</h1>
<h2>Python</h2>
<blockquote>
<p>Pexpect was written and tested with Python 2.4. It should work on
earlier versions that have the <span class="code">pty</span> module. I
sometimes even manually test it with Python 1.5.2, but I can't easily
run the PyUnit test framework against Python 1.5.2, so I have less
confidence in Pexpect on Python 1.5.2.</p>
</blockquote>
<h2>pty module</h2>
<blockquote>
<p>Any POSIX system (UNIX) with a working <span class="code">pty</span>
module should be able to run Pexpect. The <span class="code">pty</span>
module is part of the Standard Python Library, so if you are running on
a POSIX system you should have it. The <span class="code">pty</span>
module does not run the same on all platforms. It should be solid on Linux
and BSD systems. I have taken effort to try to smooth the wrinkles out of the different platforms. To learn more
about the wrinkles see <a href="#bugs">Bugs</a> and <a href="#testing">Testing</a>.</p>
</blockquote>
<p>Pexpect does not currently work on the standard Windows Python (see
the pty requirement); however, it seems to work fine using <a
href="http://www.cygwin.com/">Cygwin</a>. It is possible to build
something like a pty for Windows, but it would have to use a different
technique that I am still investigating. I know it's possible because
Libes' Expect was ported to Windows. <i>If you have any ideas or
skills to contribute in this area then I would really appreciate some
tips on how to approach this problem.</i> </p>
<hr noshade="noshade" size="1">
<h1><a name="overview"></a>Overview</h1>
<p>Pexpect can be used for automating interactive applications such as
ssh, ftp, mencoder, passwd, etc. The Pexpect interface was designed to be
easy to use. Here is an example of Pexpect in action:</p>
<blockquote>
<pre class="code"># This connects to the openbsd ftp site and<br># downloads the recursive directory listing.<br>import pexpect<br>child = pexpect.spawn ('ftp ftp.openbsd.org')<br>child.expect ('Name .*: ')<br>child.sendline ('anonymous')<br>child.expect ('Password:')<br>child.sendline ('noah@example.com')<br>child.expect ('ftp&gt; ')<br>child.sendline ('cd pub')<br>child.expect('ftp&gt; ')<br>child.sendline ('get ls-lR.gz')<br>child.expect('ftp&gt; ')<br>child.sendline ('bye')<br></pre>
</blockquote>
<p> Obviously you could write an ftp client using Python's own <span
class="code">ftplib</span> module, but this is just a demonstration.
You can use this technique with any application. This is especially
handy if you are writing automated test tools.</p>
<p>There are two important methods in Pexpect -- <span class="code"><b>expect()</b></span>
and <span class="code"><b>send()</b></span> (or <span class="code">sendline()</span>
which is like <span class="code">send()</span> with a linefeed).
The <span class="code">expect()</span> method waits for the child application
to return a given string. The string you specify is a regular expression, so
you can match complicated patterns. The <span class="code"><b>send()</b></span> method
writes a string to the child application. From the child's point of
view it looks just like someone typed the text from a terminal. After
each call to <span class="code"><b>expect()</b></span> the <span
class="code"><b>before</b></span> and <span class="code"><b>after</b></span>
properties will be set to the text printed by child application. The <span
class="code"><b>before</b></span> property will contain all text up to
the expected string pattern. The <span class="code"><b>after</b></span> string
will contain the text that was matched by the expected pattern.
The <span class="code">match</span> property is set to the <span class="code">re MatchObject</span>.
</p>
<p>An example of Pexpect in action may make things more clear. This example uses
<span class="code">ftp</span> to login to the OpenBSD site; list files
in a directory; and then pass interactive control of the ftp session to
the human user.</p>
<blockquote>
<pre class="code">import pexpect<br>child = pexpect.spawn ('ftp ftp.openbsd.org')<br>child.expect ('Name .*: ')<br>child.sendline ('anonymous')<br>child.expect ('Password:')<br>child.sendline ('noah@example.com')<br>child.expect ('ftp&gt; ')<br>child.sendline ('ls /pub/OpenBSD/')<br>child.expect ('ftp&gt; ')<br>print child.before # Print the result of the ls command.<br>child.interact() # Give control of the child to the user.<br></pre>
</blockquote>
<h2>Special EOF and TIMEOUT patterns</h2>
<p>
There are two special patterns to match the End Of File or a Timeout condition.
You you can pass these patterns to <span class="code">expect()</span>.
These patterns are not regular expressions. Use them like predefined constants.
</p>
<p>If the child has died and you have read all the child's output then ordinarily
<span class="code">expect()</span> will raise an <span class="code">EOF</span>
exception. You can read everything up to the EOF without generating an
exception by using the EOF pattern <span class="code">expect(pexpect.EOF)</span>.
In this case everything the child has output will be available in the <span
class="code">before</span> property.</p>
<p>The pattern given to <span class="code">expect()</span> may be a
regular expression or it may also be a <b>list</b> of regular expressions.
This allows you to match multiple optional responses. The <span class="code">expect()</span>
method returns the index of the pattern that was matched. For example,
say you wanted to login to a server. After entering a password you
could get various responses from the server -- your password could be
rejected; or you could be allowed in and asked for your terminal type;
or you could be let right in and given a command prompt. The following
code fragment gives an example of this:</p>
<blockquote>
<pre class="code">child.expect('password:')<br>child.sendline (my_secret_password)<br># We expect any of these three patterns...<br>i = child.expect (['Permission denied', 'Terminal type', '[#\$] '])<br>if i==0:<br> print 'Permission denied on host. Can't login'<br> child.kill(0)<br>elif i==2:<br> print 'Login OK... need to send terminal type.'<br> child.sendline('vt100')<br> child.expect ('[#\$] ')<br>elif i==3:<br> print 'Login OK.'<br> print 'Shell command prompt', child.after</pre>
</blockquote>
<p>If nothing matches an expected pattern then expect will eventually
raise a TIMEOUT exception. The default time is 30 seconds, but you can
change this by passing a timeout argument to expect():</p>
<blockquote>
<pre class="code"># Wait no more than 2 minutes (120 seconds) for password prompt.<br>child.expect('password:', timeout=120)</pre>
</blockquote>
<h2>Find the end of line -- CR/LF conventions<br>
Matching at the end of a line can be tricky<br>
$ regex pattern is useless.<br>
</h2>
<p>Pexpect matches regular expressions a little differently than what
you might be used to.
</p>
<p><i><b>The $ pattern for end of line match is useless</b></i>.
The $ matches the end of string, but Pexpect reads from the child
one character at a time, so each character looks like the end of a line.
Pexpect can't do a look-ahead into the child's output stream.
In general you would have this situation when using regular expressions
with any stream.<br>
<i>Note, pexpect does have an internal buffer, so reads are faster
than one character at a time, but from the user's perspective the regex
patterns test happens one character at a time.</i></p>
<p>The best way to match the end of a line is to look for the
newline: "\r\n" (CR/LF). Yes, that does appear to be DOS-style.
It may surprise some UNIX people to learn that terminal TTY device drivers
(dumb, vt100, ANSI, xterm, etc.) all use the CR/LF combination to signify
the end of line. Pexpect uses a Pseudo-TTY device to talk to the child application, so
when the child app prints "\n" you actually see "\r\n".
</p>
<p><b>UNIX uses just linefeeds to end lines of text, but not when it
comes to TTY devices!</b> TTY devices are more like the Windows world.
Each line of text end with a CR/LF combination. When you intercept data
from a UNIX command from a TTY device you will find that the TTY device
outputs a CR/LF combination. A UNIX command may only write a linefeed
(\n), but the TTY device driver converts it to CR/LF. This means that
your terminal will see lines end with CR/LF (hex&nbsp;<span class="code">0D&nbsp;0A</span>).
Since Pexpect emulates a terminal, to match ends of lines you have to
expect the CR/LF combination.</p>
<blockquote>
<p class="code">child.expect ('\r\n')</p>
</blockquote>
<p>If you just need to skip past a new line then <span class="code">expect
('\n')</span> by itself will work, but if you are expecting a specific
pattern before the end of line then you need to explicitly look for the
\r. For example the following expects a word at the end of a line:</p>
<blockquote>
<p class="code">child.expect ('\w+\r\n')</p>
</blockquote>
<p>But the following would both fail:</p>
<blockquote>
<p class="code">child.expect ('\w+\n')</p>
</blockquote>
<p>And as explained before, trying to use '$' to match the end of line
would not work either:</p>
<blockquote>
<p class="code">child.expect ('\w+$')</p>
</blockquote>
<p>So if you need to explicitly look for the END OF LINE, you want to
look for the CR/LF combination -- not just the LF and not the $ pattern.</p>
<p>This problem is not limited to Pexpect. This problem happens any
time you try to perform a regular expression match on a stream. Regular
expressions need to look ahead. With a stream it is hard to look ahead
because the process generating the stream may not be finished. There is no
way to know if the process has paused momentarily or is finished and
waiting for you. <font color="#cc0000">Pexpect must implicitly always
do a NON greedy match (minimal) at the end of a input {### already said
this}.</font> </p>
<p>Pexpect compiles all regular expressions with the DOTALL flag. With
the DOTALL flag a "." will match a newline. See the Python <a
href="http://www.python.org/doc/current/lib/node115.html#l2h-733">documentation</a></p>
<h2>Beware of + and * at the end of input.</h2>
<p>Remember that any time you try to match a pattern that needs
look-ahead that you will always get a minimal match (non greedy). For
example, the following will always return just one character:</p>
<blockquote>
<p class="code">child.expect ('.+')</p>
</blockquote>
<p>This example will match successfully, but will always return no
characters:</p>
<blockquote>
<p class="code">child.expect ('.*')</p>
</blockquote>
<p>Generally any star * expression will match as little as possible</p>
<p>One thing you can do is to try to force a non-ambiguous character at
the end of your <span class="code">\d+</span> pattern. Expect that
character to delimit the string. For example, you might try making the
end of your pattrn be <span class="code">\D+</span> instead of <span
class="code">\D*</span>. That means number digits alone would not
satisfy the (<span class="code">\d+</span>) pattern. You would need
some number(s) and at least one <span class="code">\D</span> at the
end. </p>
<h2>Matching groups</h2>
<p>You can group regular expression using parenthesis. After a match,
the <span class="code">match</span> parameter of the spawn object will
contain the Python Match object. </p>
<h2>Examples</h2>
<p>Using "match" and groups...</p>
<h2>Debugging</h2>
<p>If you get the string value of a pexpect.spawn object you will get
lots of useful debugging information. For debugging it's very useful to
use the following pattern:</p>
<p>try:<br>
&nbsp;&nbsp;&nbsp; i = child.expect ([pattern1, pattern2, pattern3,
etc])<br>
except:<br>
&nbsp;&nbsp;&nbsp; print "Exception was thrown"<br>
&nbsp;&nbsp;&nbsp; print "debug information:"<br>
&nbsp;&nbsp;&nbsp; print str(child)<br>
</p>
<p>It is also useful to log the child's input and out to a file or the
screen. The following will turn on logging and send output to stdout
(the screen).<br>
</p>
<p>&nbsp;&nbsp;&nbsp; child = pexpect.spawn (foo)<br>
&nbsp;&nbsp;&nbsp; child.logfile = sys.stdout<br>
<br>
</p>
<hr noshade="noshade" size="1">
<h1>Exceptions</h1>
<p><b>EOF</b></p>
<p>Note that two flavors of EOF Exception may be thrown. They are
virtually identical except for the message string. For practical
purposes you should have no need to distinguish between them, but they
do give a little extra information about what type of platform you are
running. The two messages are:</p>
<blockquote>
<p class="code">End Of File (EOF) in read(). Exception style platform.</p>
<p class="code">End Of File (EOF) in read(). Empty string style
platform.</p>
</blockquote>
<p>Some UNIX platforms will throw an exception when you try to read
from a file descriptor in the EOF state. Other UNIX platforms instead
quietly return an empty string to indicate that the EOF state has been
reached.</p>
<p><b>Expecting EOF</b></p>
<p>If you wish to read up to the end of the child's output without
generating an <span class="code">EOF</span> exception then use the <span
class="code">expect(pexpect.EOF)</span> method.</p>
<p><b>TIMEOUT</b></p>
<p>The <span class="code">expect()</span> and <span class="code">read()</span>
methods will also timeout if the child does not generate any output for
a given amount of time. If this happens they will raise a <span
class="code">TIMEOUT</span> exception. You can have these method
ignore a timeout and block indefinitely by passing None for the timeout
parameter.</p>
<blockquote>
<p class="code">child.expect(pexpect.EOF, timeout=None)</p>
</blockquote>
<hr noshade="noshade" size="1">
<h1><a name="faq"></a>FAQ</h1>
<p><b>Q: Why don't shell pipe and redirect (| and >) work when I
spawn a command?</b></p>
<p>
A: Remember that Pexpect does NOT interpret shell meta characters such as
redirect, pipe, or wild cards (&gt;, |, or *). That's done by a shell not the
command you are spawning. This is a common mistake. If you want to run a
command and pipe it through another command then you must also start a shell.
For example:
<pre>
child = pexpect.spawn('/bin/bash -c "ls -l | grep LOG &gt; log_list.txt"')
child.expect(pexpect.EOF)
</pre>
The second form of spawn (where you pass a list of arguments) is useful in
situations where you wish to spawn a command and pass it its own argument list.
This can make syntax more clear. For example, the following is equivalent to
the previous example:
<pre>
shell_cmd = 'ls -l | grep LOG &gt; log_list.txt'
child = pexpect.spawn ('/bin/bash', ['-c', shell_cmd])
child.expect (pexpect.EOF)
</pre>
</p>
<p><b>Q: Isn't there already a Python Expect?</b></p>
<p>A: Yes, there are several of them. They usually require you to
compile C. I wanted something that was pure Python and preferably a
single module that was simple to install. I also wanted something that
was easy to use. This pure Python expect only recently became possible
with the introduction of the pty module in the standard Python library.
Previously C extensions were required.</p>
<p><strong>Q: The before and after properties sound weird.</strong></p>
<p>Originally I was going to model Pexpect more after Expect, but then
I found that I could never remember how to get the context of the stuff
I was trying to parse. I hate having to read my own documentation. I
decided that it was easier for me to remember what before and after
was. It just so happens that this is how the -B and -A options in grep
works, so that made it even easier for me to remember. Whatever makes
my life easier is what's best.</p>
<p><b>Q: Why not just use Expect?</b></p>
<p>A: I love it. It's great. I has bailed me out of some real jams, but
I wanted something that would do 90% of what I need from Expect; be 10%
of the size; and allow me to write my code in Python instead of TCL.
Pexpect is not nearly as big as Expect, but Pexpect does everything I
have ever used Expect for.
<!-- :-P If I liked TCL then you wouldn't be reading this. My appologies to Don Libes -- Expect is cool, TK is cool, but TCL is only slightly better than Perl in my book. Hopefully after Expyct is done I will not need to use Expect anymore -- except for that lovely autoexpect tool. Damn, I wish I had that! --> </p>
<p><b>Q: Why not just use a pipe (popen())?</b></p>
<p>A: A pipe works fine for getting the output to non-interactive
programs. If you just want to get the output from <span class="code">ls</span>,
<span class="code">uname</span>, or <span class="code">ping</span>
then this works. Pipes do not work very well for interactive programs
and pipes will almost certainly fail for most applications that ask for
passwords such as telnet, ftp, or ssh.</p>
<p>There are two reasons for this. </p>
<p>First an application may bypass stdout and print directly to its
controlling TTY. Something like SSH will do this when it asks you for a
password. This is why you cannot redirect the password prompt because
it does not go through stdout or stderr.</p>
<p>The second reason is because most applications are built using the C
Standard IO Library (anything that uses <span class="code">#include
&lt;stdio.h&gt;</span>). One of the features of the stdio library is
that it buffers all input and output. Normally output is <b><i>line
buffered</i></b> when a program is printing to a TTY (your terminal
screen). Every time the program prints a line-feed the currently
buffered data will get printed to your screen. The problem comes when
you connect a pipe. The stdio library is smart and can tell that it is
printing to a pipe instead of a TTY. In that case it switches from line
buffer mode to <i><b>block buffered</b></i>. In this mode the
currently buffered data is flushed when the buffer is full. This causes
most interactive programs to deadlock. Block buffering is more
efficient when writing to disks and pipes. Take the situation where a
program prints a message "Enter your user name:\n" and then waits for
you type type something. In block buffered mode, the stdio library will
not put the message into the pipe even though a linefeed is printed.
The result is that you never receive the message, yet the child
application will sit and wait for you to type a response. Don't confuse
the stdio lib's buffer with the pipe's buffer. The pipe buffer is
another area that can cause problems. You could flush the input side of
a pipe, whereas you have no control over the stdio library buffer. </p>
<p>More information: the Standard IO library has three states for a
FILE *. These are: _IOFBF for block buffered; _IOLBF for line buffered;
and _IONBF for unbuffered. The STDIO lib will use block buffering when
talking to a block file descriptor such as a pipe. This is usually not
helpful for interactive programs. Short of recompiling your program to
include fflush() everywhere or recompiling a custom stdio library there
is not much a controlling application can do about this if talking over
a pipe.</p>
<p> The program may have put data in its output that remains unflushed
because the output buffer is not full; then the program will go and
deadlock while waiting for input -- because you never send it any
because you are still waiting for its output (still stuck in the
STDIO's output buffer).</p>
<p>The answer is to use a pseudo-tty. A TTY device will force <i><b>line</b></i>
buffering (as opposed to block buffering). Line buffering means that
you will get each line when the child program sends a line feed. This
corresponds to the way most interactive programs operate -- send a line
of output then wait for a line of input.</p>
<p>I put "answer" in quotes because it's ugly solution and because
there is no POSIX standard for pseudo-TTY devices (even though they
have a TTY standard...). What would make more sense to me would be to
have some way to set a mode on a file descriptor so that it will tell
the STDIO to be line-buffered. I have investigated, and I don't think
there is a way to set the buffered state of a child process. The STDIO
Library does not maintain any external state in the kernel or whatnot,
so I don't think there is any way for you to alter it. I'm not quite
sure how this line-buffered/block-buffered state change happens
internally in the STDIO library. I think the STDIO lib looks at the
file descriptor and decides to change behavior based on whether it's a
TTY or a block file (see isatty()).</p>
<p>I hope that this qualifies as helpful.</p>
<h1>Don't use a pipe to control another application...</h1>
<p>Pexpect may seem similar to <span class="code">os.popen()</span> or
<span class="code">commands</span> module. The main difference is that
Pexpect (like Expect) uses a pseudo-TTY to talk to the child
application. Most applications do no work well through the system()
call or through pipes. And probably all applications that ask a user to
type in a password will fail. These applications bypass the stdin and
read directly from the TTY device. Many applications do not explicitly
flush their output buffers. This causes deadlocks if you try to control
an interactive application using a pipe. What happens is that most UNIX
applications use the stdio (#include &lt;stdio.h&gt;) for input and
output. The stdio library behaves differently depending on where the
output is going. There is no way to control this behavior from the
client end.<br>
</p>
<p><b>Q: Can I do screen scraping with this thing?</b></p>
<p>A: That depends. If your application just does line-oriented output
then this is easy. If it does screen-oriented output then it may work,
but it could be hard. For example, trying to scrape data from the 'top'
command would be hard. The top command repaints the text window. </p>
<p>I am working on an ANSI / VT100 terminal emulator that will have
methods to get characters from an arbitrary X,Y coordinate of the
virtual screen. It works and you can play with it, but I have no
working examples at this time.</p>
<hr noshade="noshade" size="1">
<h1><a name="bugs"></a>Bugs</h1>
<h2>Threads</h2>
<p>On Linux (RH 8) you cannot spawn a child from a different thread and
pass the handle back to a worker thread. The child is successfully
spawned but you can't interact with it. The only way to make it work is
to spawn and interact with the child all in the same thread. [Adam
Kerrison] </p>
<h2><a name="echo_bug"></a>Timing issue with send() and sendline()</h2>
<p>This problem has been addressed and should not effect most users.</p>
<p>It is sometimes possible to read an echo of the string sent with <span
class="code">send()</span> and <span class="code">sendline()</span>.
If you call <span class="code">sendline()</span> and then immediately
call <span class="code">readline()</span> you may get part of your
output echoed back. You may read back what you just wrote even if the
child application does not explicitly echo it. Timing is critical. This
could be a security issue when talking to an application that asks for
a password; otherwise, this does not seem like a big deal. <i>But why
do TTYs do this</i>?</p>
<p>People usually report this when they are trying to control SSH or
some other login. For example, if your code looks something like this: </p>
<pre class="code">child.expect ('[pP]assword:')<br>child.sendline (my_password)</pre>
<p><br>
<blockquote>
1. SSH prints "password:" prompt to the user.<br>
2. SSH turns off echo on the TTY device.<br>
3. SSH waits for user to enter a password.<br>
</blockquote>
When scripting with Pexpect what can happen is that Pexpect will response to the "password:" prompt
before SSH has had time to turn off TTY echo. In other words, Pexpect sends the password between
steps 1. and 2., so the password gets echoed back to the TTY. I would call this an SSH bug.
</p>
<p>
Pexpect now automatically adds a short delay before sending data to a child process.
This more closely mimics what happens in the usual human-to-app interaction.
The delay can be tuned with the 'delaybeforesend' attribute of the spawn class.
In general, this fixes the problem for everyone and so this should not be an issue
for most users. For some applications you might with to turn it off.
child = pexpect.spawn ("ssh user@example.com")
child.delaybeforesend = 0
</p>
<p><br>
</p>
<p>Try changing it to look like the following. I know that this fix
does not look correct, but it works. I have not figured out exactly
what is happening. You would think that the sleep should be after the
sendline(). The fact that the sleep helps when it's between the
expect() and the sendline() must be a clue.</p>
<pre class="code">child.expect ('[pP]assword:')<br>child.sendline (my_password)</pre>
<h2>Timing issue with isalive()</h2>
<p>Reading the state of isalive() immediately after a child exits may
sometimes return 1. This is a race condition. The child has closed its
file descriptor, but has not yet fully exited before Pexpect's
isalive() executes. Addings a slight delay before the isalive() will
help. In the following example <span class="code">isalive()</span>
sometimes returns 1:</p>
<blockquote>
<pre class="code">child = pexpect.spawn('ls')<br>child.expect(pexpect.EOF)<br>print child.isalive()</pre>
</blockquote>
<p>But if there is any delay before the call to <span class="code">isalive()</span>
then it will always return 0 as expected.</p>
<blockquote>
<pre class="code">child = pexpect.spawn('ls')<br>child.expect(pexpect.EOF)<br>time.sleep(0.1)<br>print child.isalive()</pre>
</blockquote>
<h2>Truncated output just before child exits</h2>
<p><i>So far I have seen this only on older versions of <b>Apple's MacOS X</b>.</i>
If the child application quits it may not flush its output buffer. This
means that your Pexpect application will receive an EOF even though it
should have received a little more data before the child died. This is
not generally a problem when talking to interactive child applications.
One example where it is a problem is when trying to read output from a
program like '<span class="code">ls</span>'. You may receive most of
the directory listing, but the last few lines will get lost before you
receive an EOF. The reason for this is that '<span class="code">ls</span>'
runs; completes its task; and then exits. The buffer is not flushed
before exit so the last few lines are lost. The following example
demonstrates the problem:</p>
<p> </p>
<blockquote>
<pre class="code">child = pexpect.spawn ('ls -l')<br>child.expect (pexpect.EOF)<br>print child.before <br> </pre>
</blockquote>
<p></p>
<h2>Controlling SSH on Solaris</h2>
<p>Pexpect does not yet work perfectly on Solaris.
One common problem is that SSH sometimes will not allow TTY password
authentication. For example, you may expect SSH to ask you for a
password using code like this:
</p>
<pre class="code">child = pexpect.spawn ('ssh user@example.com')<br>child.expect ('assword')<br>child.sendline ('mypassword')<br></pre>
You may see the following error come back from a spawned
child SSH:
<p></p>
<blockquote>Permission denied (publickey,keyboard-interactive). </blockquote>
<p>
This means that SSH thinks it can't access the TTY to ask you for your
password.
The only solution I have found is to use public key authentication with
SSH.
This bypasses the need for a password. I'm not happy with this
solution.
The problem is due to poor support for Solaris Pseudo TTYs in the
Python
Standard Library. </p>
<hr noshade="noshade" size="1">
<h1><a name="changes"></a>CHANGES</h1>
<h2>Current Release</h2>
<p>Fixed OSError exception when a pexpect object is cleaned up.
Previously you might have seen this exception:</p>
<blockquote>
<pre class="code">Exception exceptions.OSError: (10, 'No child processes') <br>in &lt;bound method spawn.__del__ of<br>&lt;pexpect.spawn instance at 0xd248c&gt;&gt; ignored</pre>
</blockquote>
<p>You should not see that anymore. Thanks to Michael Surette.</p>
<p>Added support for buffering reads. This greatly improves speed when
trying to match long output from a child process. When you create an
instance of the spawn object you can then set a buffer size. For now
you MUST do the following to turn on buffering -- it may be on by
default in future version.</p>
<blockquote>
<pre class="code">child = pexpect.spawn ('my_command')<br>child.maxread=1000 # Sets buffer to 1000 characters.</pre>
</blockquote>
<div>
<p>I made a subtle change to the way TIMEOUT and EOF exceptions behave.
Previously you could either expect these states in which case pexpect
will not raise an exception, or you could just let pexpect raise an
exception when these states were encountered. If you expected the
states then the 'before' property was set to everything before the
state was encountered, but if you let pexpect raise the exception then
'before' was not set. Now the 'before' property will get set either way
you choose to handle these states.</p>
<h2><i>Older changes...</i></h2>
<p>The spawn object now provides iterators for a <i>file-like interface</i>.
This makes Pexpect a more complete file-like object. You can now write
code like this:</p>
<blockquote>
<pre class="code">child = pexpect.spawn ('ls -l')<br>for line in child:<br> print line<br></pre>
</blockquote>
<p>I added the attribute <span class="code">exitstatus</span>. This
will give the exit code returned by the child process. This will be set
to <span class="code">None</span> while the child is still alive. When
<span class="code">isalive()</span> returns 0 then <span class="code">exitstatus</span>
will be set.</p>
<p>I made a few more tweaks to <span class="code">isalive()</span> so
that it will operate more consistently on different platforms. Solaris
is the most difficult to support.</p>
<p>&nbsp;</p>
<p>You can now put <span class="code">TIMEOUT</span> in a list of
expected patterns. This is just like putting <span class="code">EOF</span>
in the pattern list. Expecting for a <span class="code">TIMEOUT</span>
may not be used as often as <span class="code">EOF</span>, but this
makes Pexpect more consitent.</p>
<p>Thanks to a suggestion and sample code from Chad J. Schroeder I
added the ability for Pexpect to operate on a file descriptor that is
already open. This means that Pexpect can be used to control streams
such as those from serial port devices. Now you just pass the integer
file descriptor as the "command" when contsructing a spawn open. For
example on a Linux box with a modem on ttyS1:</p>
<blockquote>
<pre class="code">fd = os.open("/dev/ttyS1", os.O_RDWR|os.O_NONBLOCK|os.O_NOCTTY)<br>m = pexpect.spawn(fd) # Note integer fd is used instead of usual string.<br>m.send("+++") # Escape sequence<br>m.send("ATZ0\r") # Reset modem to profile 0<br>rval = m.expect(["OK", "ERROR"])</pre>
</blockquote>
<h3>Pexpect now tests itself on Compile Farm!</h3>
<p>I wrote a nice script that uses ssh to connect to each machine on
Source Forge's Compile Farm and then run the testall.py script for each
platform. The result of the test is then recorded for each platform.
Now it's easy to run regression tests across multiple platforms.</p>
<h3>Pexpect is a file-like object</h3>
<p>The spawn object now provides a <i>file-like interface</i>. It
supports most of the methods and attributes defined for Python File
Objects. </p>
<p>I changed write and writelines() so that they no longer return a
value. Use send() if you need that functionality. I did this to make
the Spawn object more closely match a file-like object.</p>
<p>read() was renamed to read_nonblocking(). I added a new read()
method that matches file-like object interface. In general, you should
not notice the difference except that read() no longer allows you to
directly set the timeout value. I hope this will not effect any
existing code. Switching to read_nonblocking() should fix existing code.</p>
<p>I changed the name of <span class="code">set_echo()</span> to <span
class="code">setecho()</span>.</p>
<p>I changed the name of <span class="code">send_eof()</span> to <span
class="code">sendeof()</span>.</p>
<p>I modified <span class="code">kill()</span> so that it checks to
make sure the pid isalive().</p>
<p>I modified <span class="code">spawn()</span> (really called from <span
class="code">__spawn()</span>)so that it does not raise an expection
if <span class="code">setwinsize()</span> fails. Some platforms such
as Cygwin do not like setwinsize. This was a constant problem and since
it is not a critical feature I decided to just silence the error.
Normally I don't like to do that, but in this case I'm making an
exception.</p>
<p>Added a method <span class="code">close()</span> that does what you
think. It closes the file descriptor of the child application. It makes
no attempt to actually kill the child or wait for its status. </p>
<p>Add variables <span class="code">__version__</span> and <span
class="code">__revision__</span> (from cvs) to the pexpect modules.
This is mainly helpful to me so that I can make sure that I'm testing
with the right version instead of one already installed.</p>
<h3>Logging changes</h3>
<blockquote>
<p><span class="code">log_open()</span> and <span class="code">log_close()</span>
have been removed. Now use <span class="code">setlog()</span>. The <span
class="code">setlog()</span> method takes a file object. This is far
more flexible than the previous log method. Each time data is written
to the file object it will be flushed. To turn logging off simply call <span
class="code">setlog()</span> with None.</p>
</blockquote>
<h2>isalive changes</h2>
<blockquote>
<p>I renamed the <span class="code">isAlive()</span> method to <span
class="code">isalive()</span> to match the more typical naming style
in Python. Also the technique used to detect child process status has
been drastically modified. Previously I did some funky stuff with
signals which caused indigestion in other Python modules on some
platforms. It's was a big headache. It still is, but I think it works
better now.</p>
</blockquote>
<h3>attribute name changes</h3>
<blockquote>
<p>The names of some attributes have been changed. This effects the
names of the attributes that are set after called the <span
class="code">expect()</span> method.</p>
<table class="pymenu" border="0" cellpadding="5">
<tbody>
<tr>
<th class="pymenu">NEW NAME</th>
<th class="pymenu">OLD NAME</th>
</tr>
<tr>
<td><span class="code">before</span><br>
<i>Everything before the match.</i></td>
<td><span class="code">before</span></td>
</tr>
<tr>
<td><span class="code">after</span><br>
<i>Everything after and including the first character of the
match</i></td>
<td><span class="code">matched</span></td>
</tr>
<tr>
<td><span class="code">match</span><br>
<i>This is the re MatchObject from the match.<br>
You can get groups() from this.<br>
See '<span class="code">uptime.py</span>' in the examples tar ball.</i></td>
<td><i>New -- Did not exist</i></td>
</tr>
</tbody>
</table>
</blockquote>
<h3>EOF changes</h3>
<blockquote>
<p>The <span class="code">expect_eof()</span> method is gone. You
can now simply use the <span class="code">expect()</span> method to
look for EOF.</p>
<p>Was:</p>
<blockquote>
<p><span class="code">p.expect_eof ()</span></p>
</blockquote>
<p>Now:</p>
<blockquote>
<p><span class="code">p.expect (pexpect.EOF)</span></p>
</blockquote>
</blockquote>
<hr noshade="noshade" size="1">
<h1><a name="testing"></a>TESTING</h1>
<p>The following platforms have been tested:</p>
<!--
<table class="pymenu" border="0" cellpadding="5">
<tbody>
<tr>
<th class="pymenu">PLATFORM</th>
<th class="pymenu">RESULTS</th>
</tr>
<tr>
<td>Linux 2.4.9-ac10-rmk2-np1-cerf2<br>
armv4l</td>
<td><b><i>all tests passed</i></b></td>
</tr>
<tr>
<td>Linux 2.4.18 #2<br>
sparc64</td>
<td><b><i>all tests passed</i></b></td>
</tr>
<tr>
<td>MacOS X Darwin Kernel Version 5.5<br>
powerpc</td>
<td>
<p>failed more than one test.</p>
<p>Generally Pexpect works on OS X, but the nature of the quirks
cause a many of the tests to fail. See <a href="#bugs">bugs</a>
(Incomplete Child Output). The problem is more than minor, but Pexpect
is still more than useful for most tasks. The problem is an edge case.</p>
</td>
</tr>
<tr>
<td>Linux 2.2.20<br>
alpha<br>
</td>
<td><b><i>all tests passed</i></b></td>
</tr>
<tr>
<td>Linux 2.4.18-5smp<br>
i686</td>
<td><b><i>all tests passed</i></b></td>
</tr>
<tr>
<td>OpenBSD 2.9 GENERIC#653<br>
i386</td>
<td><b><i>all tests passed</i></b></td>
</tr>
<tr>
<td>Solaris</td>
<td>
<p>failed <span class="code">test_destructor</span></p>
<p>Otherwise, this is working pretty well. The destructor problem
is minor. For some reason, the <i>second</i> time a pty file
descriptor is created and deleted it never gets returned for use. It
does not effect the first time or the third time or any time after
that. It's only the second time. This is weird... This could be a file
descriptor leak, or it could be some peculiarity of how Solaris
recycles them. I thought it was a UNIX requirement for the OS to give
you the lowest available filedescriptor number. In any case, this
should not be a problem unless you create hundreds of pexpect
instances... It may also be a pty module bug. </p>
</td>
</tr>
<tr>
<td>Windows XP Cygwin</td>
<td>failed <span class="code">test_destructor</span>. That it
works at all is amazing to me. Cygwin rules!</td>
</tr>
</tbody>
</table>
-->
<h1>&nbsp;</h1>
<h1><a name="todo">TO DO</a></h1>
<p>Add an option to add a delay after each expect() or before each
read()/readline() call to automatically avoid the <a href="#echo_bug">echo
bug</a>.</p>
<p>&nbsp;</p>
</div>
<hr noshade="noshade" size="1">
<table border="0">
<tbody>
<tr>
<td> <a href="http://www.noah.org/email/"><img src="email.png"
alt="Click to send email." border="0" height="16" width="100"></a> </td>
</tr>
</tbody>
</table>
</div>
<div id="Menu"><b>INDEX</b><br>
<hr noshade="noshade" size="1"> <a href="#license"
title="Python Software Foundation License">License</a><br>
<a href="#download" title="Download and setup instructions">Download</a><br>
<a href="#doc" title="Documentation and overview">Documentation</a><br>
<a href="#status" title="Project Status">Project Status</a><br>
<a href="#requirements" title="System requirements to use Pexpect">Requirements</a><br>
<a href="#overview" title="Overview of what Pexpect does">Overview</a><br>
<a href="#faq" title="FAQ">FAQ</a><br>
<a href="#bugs" title="Bugs and work-arounds">Known Bugs</a><br>
<a href="#changes" title="What's new with Pexpect">Recent Changes</a><br>
<a href="#testing" title="Test results on various platforms">Testing</a><br>
<a href="#todo" title="What to do next">To do</a><br>
<a href="http://pexpect.svn.sourceforge.net/viewvc/pexpect/trunk/pexpect/" title="browse SVN">Browse SVN</a><br>
<br>
<a href="http://sourceforge.net/projects/pexpect/"
title="The Pexpect project page on SourceForge.net"> <img
src="http://sourceforge.net/sflogo.php?group_id=59762&amp;type=5"
alt="The Pexpect project page on SourceForge.net" border="0"
height="31" width="105"> </a> </div>
</body>
</html>

View File

@ -0,0 +1,868 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<title>Pexpect - a Pure Python Expect-like module</title>
<link rel="stylesheet" href="clean.css" type="text/css">
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<meta name="Author" content="Noah Spurrier">
<meta name="Keywords"
content="pexpect, Noah Spurrier, pypect, Python, Libes, TCL, Expect, pipe, popen, pyExpect, expectpy, expect-like, expect-alike, expect like">
<meta name="Description"
content="Pexpect is a pure Python Expect-like module. Pexpect makes Python a better tool for controlling other applications.">
</head>
<body bgcolor="#ffffff" text="#000000">
<div id="Header">
<h1>Pexpect version VERSION<br>
a Pure Python Expect-like module
</h1>
</div>
<div id="Content">
<p>Pexpect makes Python a better tool for controlling other
applications.</p>
<p>Pexpect is a pure Python module for spawning child applications;
controlling them; and responding to expected patterns in their output.
Pexpect works like Don Libes' Expect. Pexpect allows your script to
spawn a child application and control it as if a human were typing
commands.</p>
<p>Pexpect can be used for automating interactive applications such as
ssh, ftp, passwd, telnet, etc. It can be used to a automate setup
scripts for duplicating software package installations on different
servers. It can be used for automated software testing. Pexpect is in
the spirit of Don Libes' Expect, but Pexpect is pure Python. Unlike
other Expect-like modules for Python, Pexpect does not require TCL or
Expect nor does it require C extensions to be compiled. It should work
on any platform that supports the standard Python pty module. The
Pexpect interface was designed to be easy to use.</p>
<table border="0">
<tbody>
<tr>
<td align="right" valign="top">Send questions to:</td>
<td align="left"><a href="http://www.noah.org/email/"><img
src="email.png" alt="Click to send email." border="0" height="16"
width="100"></a></td>
</tr>
</tbody>
</table>
<hr noshade="noshade" size="1">
<h1><a name="license"></a>License: MIT style</h1>
<p>
Free, open source, and all that good stuff.<br>
<br>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:<br>
<br>
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.<br>
<br>
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.<br>
<br>
Pexpect Copyright (c) 2008 Noah Spurrier<br>
http://pexpect.sourceforge.net/
</p>
<hr noshade="noshade" size="1">
<h1><a name="download"></a><a
href="http://sourceforge.net/project/showfiles.php?group_id=59762">Download</a></h1>
<p>Download the <a
href="http://sourceforge.net/project/showfiles.php?group_id=59762">
current version here</a> from the SourceForge site. Grab the current Pexpect tarball.
</p>
<h2>Installing Pexpect</h2>
<p>The Pexpect tarball is a standard Python Distutil distribution.</p>
<ol>
<li>download <span class="code">pexpect-VERSION.tar.gz</span></li>
<li><span class="code">tar zxf pexpect-VERSION.tar.gz</span></li>
<li><span class="code">cd pexpect-VERSION</span></li>
<li><span class="code">python setup.py install</span> <i>do this as root</i></li>
</ol>
<h2>Examples</h2>
<p>
Under the <span class="code">pexpect-VERSION</span> directory you should find
the <span class="code">examples</span> directory.
This is the best way to learn to use Pexpect.
See the descriptions of <a href="examples.html">Pexpect Examples</a>.
</p>
<h2><a name="doc"></a>API Documentation</h2>
<p>
<blockquote>
<a href="pexpect.html">pexpect</a> This is the main module that you want.<br>
<a href="pxssh.html">pxssh</a> Pexpect SSH is an extension of 'pexpect.spawn' that specializes in SSH.<br>
</blockquote>
the following are experimental extensions to Pexpect<br>
<blockquote>
<a href="fdpexpect.html">fdpexpect</a> fdpexpect extension of 'pexpect.spawn' that uses an open file descriptor.<br>
<a href="screen.html">SCREEN</a> This represents a virtual 'screen'.<br>
<a href="ANSI.html">ANSI</a> This parses ANSI/VT-100 terminal escape codes.<br>
<a href="FSM.html">FSM</a> This is a finite state machine used by ANSI.<br>
</blockquote>
</p>
<hr noshade="noshade" size="1">
<h1><a name="status"></a>Project Status</h1>
<p>Automated pyunit tests reach over 80%
code coverage on pexpect.py. I regularly test on Linux and BSD
platforms. I try to test on Solaris and Irix.
</p>
<hr noshade="noshade" size="1">
<h1><a name="requirements"></a>Requirements for use of Pexpect</h1>
<h2>Python</h2>
<blockquote>
<p>Pexpect was written and tested with Python 2.4. It should work on
earlier versions that have the <span class="code">pty</span> module. I
sometimes even manually test it with Python 1.5.2, but I can't easily
run the PyUnit test framework against Python 1.5.2, so I have less
confidence in Pexpect on Python 1.5.2.</p>
</blockquote>
<h2>pty module</h2>
<blockquote>
<p>Any POSIX system (UNIX) with a working <span class="code">pty</span>
module should be able to run Pexpect. The <span class="code">pty</span>
module is part of the Standard Python Library, so if you are running on
a POSIX system you should have it. The <span class="code">pty</span>
module does not run the same on all platforms. It should be solid on Linux
and BSD systems. I have taken effort to try to smooth the wrinkles out of the different platforms. To learn more
about the wrinkles see <a href="#bugs">Bugs</a> and <a href="#testing">Testing</a>.</p>
</blockquote>
<p>Pexpect does not currently work on the standard Windows Python (see
the pty requirement); however, it seems to work fine using <a
href="http://www.cygwin.com/">Cygwin</a>. It is possible to build
something like a pty for Windows, but it would have to use a different
technique that I am still investigating. I know it's possible because
Libes' Expect was ported to Windows. <i>If you have any ideas or
skills to contribute in this area then I would really appreciate some
tips on how to approach this problem.</i> </p>
<hr noshade="noshade" size="1">
<h1><a name="overview"></a>Overview</h1>
<p>Pexpect can be used for automating interactive applications such as
ssh, ftp, mencoder, passwd, etc. The Pexpect interface was designed to be
easy to use. Here is an example of Pexpect in action:</p>
<blockquote>
<pre class="code"># This connects to the openbsd ftp site and<br># downloads the recursive directory listing.<br>import pexpect<br>child = pexpect.spawn ('ftp ftp.openbsd.org')<br>child.expect ('Name .*: ')<br>child.sendline ('anonymous')<br>child.expect ('Password:')<br>child.sendline ('noah@example.com')<br>child.expect ('ftp&gt; ')<br>child.sendline ('cd pub')<br>child.expect('ftp&gt; ')<br>child.sendline ('get ls-lR.gz')<br>child.expect('ftp&gt; ')<br>child.sendline ('bye')<br></pre>
</blockquote>
<p> Obviously you could write an ftp client using Python's own <span
class="code">ftplib</span> module, but this is just a demonstration.
You can use this technique with any application. This is especially
handy if you are writing automated test tools.</p>
<p>There are two important methods in Pexpect -- <span class="code"><b>expect()</b></span>
and <span class="code"><b>send()</b></span> (or <span class="code">sendline()</span>
which is like <span class="code">send()</span> with a linefeed).
The <span class="code">expect()</span> method waits for the child application
to return a given string. The string you specify is a regular expression, so
you can match complicated patterns. The <span class="code"><b>send()</b></span> method
writes a string to the child application. From the child's point of
view it looks just like someone typed the text from a terminal. After
each call to <span class="code"><b>expect()</b></span> the <span
class="code"><b>before</b></span> and <span class="code"><b>after</b></span>
properties will be set to the text printed by child application. The <span
class="code"><b>before</b></span> property will contain all text up to
the expected string pattern. The <span class="code"><b>after</b></span> string
will contain the text that was matched by the expected pattern.
The <span class="code">match</span> property is set to the <span class="code">re MatchObject</span>.
</p>
<p>An example of Pexpect in action may make things more clear. This example uses
<span class="code">ftp</span> to login to the OpenBSD site; list files
in a directory; and then pass interactive control of the ftp session to
the human user.</p>
<blockquote>
<pre class="code">import pexpect<br>child = pexpect.spawn ('ftp ftp.openbsd.org')<br>child.expect ('Name .*: ')<br>child.sendline ('anonymous')<br>child.expect ('Password:')<br>child.sendline ('noah@example.com')<br>child.expect ('ftp&gt; ')<br>child.sendline ('ls /pub/OpenBSD/')<br>child.expect ('ftp&gt; ')<br>print child.before # Print the result of the ls command.<br>child.interact() # Give control of the child to the user.<br></pre>
</blockquote>
<h2>Special EOF and TIMEOUT patterns</h2>
<p>
There are two special patterns to match the End Of File or a Timeout condition.
You you can pass these patterns to <span class="code">expect()</span>.
These patterns are not regular expressions. Use them like predefined constants.
</p>
<p>If the child has died and you have read all the child's output then ordinarily
<span class="code">expect()</span> will raise an <span class="code">EOF</span>
exception. You can read everything up to the EOF without generating an
exception by using the EOF pattern <span class="code">expect(pexpect.EOF)</span>.
In this case everything the child has output will be available in the <span
class="code">before</span> property.</p>
<p>The pattern given to <span class="code">expect()</span> may be a
regular expression or it may also be a <b>list</b> of regular expressions.
This allows you to match multiple optional responses. The <span class="code">expect()</span>
method returns the index of the pattern that was matched. For example,
say you wanted to login to a server. After entering a password you
could get various responses from the server -- your password could be
rejected; or you could be allowed in and asked for your terminal type;
or you could be let right in and given a command prompt. The following
code fragment gives an example of this:</p>
<blockquote>
<pre class="code">child.expect('password:')<br>child.sendline (my_secret_password)<br># We expect any of these three patterns...<br>i = child.expect (['Permission denied', 'Terminal type', '[#\$] '])<br>if i==0:<br> print 'Permission denied on host. Can't login'<br> child.kill(0)<br>elif i==2:<br> print 'Login OK... need to send terminal type.'<br> child.sendline('vt100')<br> child.expect ('[#\$] ')<br>elif i==3:<br> print 'Login OK.'<br> print 'Shell command prompt', child.after</pre>
</blockquote>
<p>If nothing matches an expected pattern then expect will eventually
raise a TIMEOUT exception. The default time is 30 seconds, but you can
change this by passing a timeout argument to expect():</p>
<blockquote>
<pre class="code"># Wait no more than 2 minutes (120 seconds) for password prompt.<br>child.expect('password:', timeout=120)</pre>
</blockquote>
<h2>Find the end of line -- CR/LF conventions<br>
Matching at the end of a line can be tricky<br>
$ regex pattern is useless.<br>
</h2>
<p>Pexpect matches regular expressions a little differently than what
you might be used to.
</p>
<p><i><b>The $ pattern for end of line match is useless</b></i>.
The $ matches the end of string, but Pexpect reads from the child
one character at a time, so each character looks like the end of a line.
Pexpect can't do a look-ahead into the child's output stream.
In general you would have this situation when using regular expressions
with any stream.<br>
<i>Note, pexpect does have an internal buffer, so reads are faster
than one character at a time, but from the user's perspective the regex
patterns test happens one character at a time.</i></p>
<p>The best way to match the end of a line is to look for the
newline: "\r\n" (CR/LF). Yes, that does appear to be DOS-style.
It may surprise some UNIX people to learn that terminal TTY device drivers
(dumb, vt100, ANSI, xterm, etc.) all use the CR/LF combination to signify
the end of line. Pexpect uses a Pseudo-TTY device to talk to the child application, so
when the child app prints "\n" you actually see "\r\n".
</p>
<p><b>UNIX uses just linefeeds to end lines of text, but not when it
comes to TTY devices!</b> TTY devices are more like the Windows world.
Each line of text end with a CR/LF combination. When you intercept data
from a UNIX command from a TTY device you will find that the TTY device
outputs a CR/LF combination. A UNIX command may only write a linefeed
(\n), but the TTY device driver converts it to CR/LF. This means that
your terminal will see lines end with CR/LF (hex&nbsp;<span class="code">0D&nbsp;0A</span>).
Since Pexpect emulates a terminal, to match ends of lines you have to
expect the CR/LF combination.</p>
<blockquote>
<p class="code">child.expect ('\r\n')</p>
</blockquote>
<p>If you just need to skip past a new line then <span class="code">expect
('\n')</span> by itself will work, but if you are expecting a specific
pattern before the end of line then you need to explicitly look for the
\r. For example the following expects a word at the end of a line:</p>
<blockquote>
<p class="code">child.expect ('\w+\r\n')</p>
</blockquote>
<p>But the following would both fail:</p>
<blockquote>
<p class="code">child.expect ('\w+\n')</p>
</blockquote>
<p>And as explained before, trying to use '$' to match the end of line
would not work either:</p>
<blockquote>
<p class="code">child.expect ('\w+$')</p>
</blockquote>
<p>So if you need to explicitly look for the END OF LINE, you want to
look for the CR/LF combination -- not just the LF and not the $ pattern.</p>
<p>This problem is not limited to Pexpect. This problem happens any
time you try to perform a regular expression match on a stream. Regular
expressions need to look ahead. With a stream it is hard to look ahead
because the process generating the stream may not be finished. There is no
way to know if the process has paused momentarily or is finished and
waiting for you. <font color="#cc0000">Pexpect must implicitly always
do a NON greedy match (minimal) at the end of a input {### already said
this}.</font> </p>
<p>Pexpect compiles all regular expressions with the DOTALL flag. With
the DOTALL flag a "." will match a newline. See the Python <a
href="http://www.python.org/doc/current/lib/node115.html#l2h-733">documentation</a></p>
<h2>Beware of + and * at the end of input.</h2>
<p>Remember that any time you try to match a pattern that needs
look-ahead that you will always get a minimal match (non greedy). For
example, the following will always return just one character:</p>
<blockquote>
<p class="code">child.expect ('.+')</p>
</blockquote>
<p>This example will match successfully, but will always return no
characters:</p>
<blockquote>
<p class="code">child.expect ('.*')</p>
</blockquote>
<p>Generally any star * expression will match as little as possible</p>
<p>One thing you can do is to try to force a non-ambiguous character at
the end of your <span class="code">\d+</span> pattern. Expect that
character to delimit the string. For example, you might try making the
end of your pattrn be <span class="code">\D+</span> instead of <span
class="code">\D*</span>. That means number digits alone would not
satisfy the (<span class="code">\d+</span>) pattern. You would need
some number(s) and at least one <span class="code">\D</span> at the
end. </p>
<h2>Matching groups</h2>
<p>You can group regular expression using parenthesis. After a match,
the <span class="code">match</span> parameter of the spawn object will
contain the Python Match object. </p>
<h2>Examples</h2>
<p>Using "match" and groups...</p>
<h2>Debugging</h2>
<p>If you get the string value of a pexpect.spawn object you will get
lots of useful debugging information. For debugging it's very useful to
use the following pattern:</p>
<p>try:<br>
&nbsp;&nbsp;&nbsp; i = child.expect ([pattern1, pattern2, pattern3,
etc])<br>
except:<br>
&nbsp;&nbsp;&nbsp; print "Exception was thrown"<br>
&nbsp;&nbsp;&nbsp; print "debug information:"<br>
&nbsp;&nbsp;&nbsp; print str(child)<br>
</p>
<p>It is also useful to log the child's input and out to a file or the
screen. The following will turn on logging and send output to stdout
(the screen).<br>
</p>
<p>&nbsp;&nbsp;&nbsp; child = pexpect.spawn (foo)<br>
&nbsp;&nbsp;&nbsp; child.logfile = sys.stdout<br>
<br>
</p>
<hr noshade="noshade" size="1">
<h1>Exceptions</h1>
<p><b>EOF</b></p>
<p>Note that two flavors of EOF Exception may be thrown. They are
virtually identical except for the message string. For practical
purposes you should have no need to distinguish between them, but they
do give a little extra information about what type of platform you are
running. The two messages are:</p>
<blockquote>
<p class="code">End Of File (EOF) in read(). Exception style platform.</p>
<p class="code">End Of File (EOF) in read(). Empty string style
platform.</p>
</blockquote>
<p>Some UNIX platforms will throw an exception when you try to read
from a file descriptor in the EOF state. Other UNIX platforms instead
quietly return an empty string to indicate that the EOF state has been
reached.</p>
<p><b>Expecting EOF</b></p>
<p>If you wish to read up to the end of the child's output without
generating an <span class="code">EOF</span> exception then use the <span
class="code">expect(pexpect.EOF)</span> method.</p>
<p><b>TIMEOUT</b></p>
<p>The <span class="code">expect()</span> and <span class="code">read()</span>
methods will also timeout if the child does not generate any output for
a given amount of time. If this happens they will raise a <span
class="code">TIMEOUT</span> exception. You can have these method
ignore a timeout and block indefinitely by passing None for the timeout
parameter.</p>
<blockquote>
<p class="code">child.expect(pexpect.EOF, timeout=None)</p>
</blockquote>
<hr noshade="noshade" size="1">
<h1><a name="faq"></a>FAQ</h1>
<p><b>Q: Why don't shell pipe and redirect (| and >) work when I
spawn a command?</b></p>
<p>
A: Remember that Pexpect does NOT interpret shell meta characters such as
redirect, pipe, or wild cards (&gt;, |, or *). That's done by a shell not the
command you are spawning. This is a common mistake. If you want to run a
command and pipe it through another command then you must also start a shell.
For example:
<pre>
child = pexpect.spawn('/bin/bash -c "ls -l | grep LOG &gt; log_list.txt"')
child.expect(pexpect.EOF)
</pre>
The second form of spawn (where you pass a list of arguments) is useful in
situations where you wish to spawn a command and pass it its own argument list.
This can make syntax more clear. For example, the following is equivalent to
the previous example:
<pre>
shell_cmd = 'ls -l | grep LOG &gt; log_list.txt'
child = pexpect.spawn ('/bin/bash', ['-c', shell_cmd])
child.expect (pexpect.EOF)
</pre>
</p>
<p><b>Q: Isn't there already a Python Expect?</b></p>
<p>A: Yes, there are several of them. They usually require you to
compile C. I wanted something that was pure Python and preferably a
single module that was simple to install. I also wanted something that
was easy to use. This pure Python expect only recently became possible
with the introduction of the pty module in the standard Python library.
Previously C extensions were required.</p>
<p><strong>Q: The before and after properties sound weird.</strong></p>
<p>Originally I was going to model Pexpect more after Expect, but then
I found that I could never remember how to get the context of the stuff
I was trying to parse. I hate having to read my own documentation. I
decided that it was easier for me to remember what before and after
was. It just so happens that this is how the -B and -A options in grep
works, so that made it even easier for me to remember. Whatever makes
my life easier is what's best.</p>
<p><b>Q: Why not just use Expect?</b></p>
<p>A: I love it. It's great. I has bailed me out of some real jams, but
I wanted something that would do 90% of what I need from Expect; be 10%
of the size; and allow me to write my code in Python instead of TCL.
Pexpect is not nearly as big as Expect, but Pexpect does everything I
have ever used Expect for.
<!-- :-P If I liked TCL then you wouldn't be reading this. My appologies to Don Libes -- Expect is cool, TK is cool, but TCL is only slightly better than Perl in my book. Hopefully after Expyct is done I will not need to use Expect anymore -- except for that lovely autoexpect tool. Damn, I wish I had that! --> </p>
<p><b>Q: Why not just use a pipe (popen())?</b></p>
<p>A: A pipe works fine for getting the output to non-interactive
programs. If you just want to get the output from <span class="code">ls</span>,
<span class="code">uname</span>, or <span class="code">ping</span>
then this works. Pipes do not work very well for interactive programs
and pipes will almost certainly fail for most applications that ask for
passwords such as telnet, ftp, or ssh.</p>
<p>There are two reasons for this. </p>
<p>First an application may bypass stdout and print directly to its
controlling TTY. Something like SSH will do this when it asks you for a
password. This is why you cannot redirect the password prompt because
it does not go through stdout or stderr.</p>
<p>The second reason is because most applications are built using the C
Standard IO Library (anything that uses <span class="code">#include
&lt;stdio.h&gt;</span>). One of the features of the stdio library is
that it buffers all input and output. Normally output is <b><i>line
buffered</i></b> when a program is printing to a TTY (your terminal
screen). Every time the program prints a line-feed the currently
buffered data will get printed to your screen. The problem comes when
you connect a pipe. The stdio library is smart and can tell that it is
printing to a pipe instead of a TTY. In that case it switches from line
buffer mode to <i><b>block buffered</b></i>. In this mode the
currently buffered data is flushed when the buffer is full. This causes
most interactive programs to deadlock. Block buffering is more
efficient when writing to disks and pipes. Take the situation where a
program prints a message "Enter your user name:\n" and then waits for
you type type something. In block buffered mode, the stdio library will
not put the message into the pipe even though a linefeed is printed.
The result is that you never receive the message, yet the child
application will sit and wait for you to type a response. Don't confuse
the stdio lib's buffer with the pipe's buffer. The pipe buffer is
another area that can cause problems. You could flush the input side of
a pipe, whereas you have no control over the stdio library buffer. </p>
<p>More information: the Standard IO library has three states for a
FILE *. These are: _IOFBF for block buffered; _IOLBF for line buffered;
and _IONBF for unbuffered. The STDIO lib will use block buffering when
talking to a block file descriptor such as a pipe. This is usually not
helpful for interactive programs. Short of recompiling your program to
include fflush() everywhere or recompiling a custom stdio library there
is not much a controlling application can do about this if talking over
a pipe.</p>
<p> The program may have put data in its output that remains unflushed
because the output buffer is not full; then the program will go and
deadlock while waiting for input -- because you never send it any
because you are still waiting for its output (still stuck in the
STDIO's output buffer).</p>
<p>The answer is to use a pseudo-tty. A TTY device will force <i><b>line</b></i>
buffering (as opposed to block buffering). Line buffering means that
you will get each line when the child program sends a line feed. This
corresponds to the way most interactive programs operate -- send a line
of output then wait for a line of input.</p>
<p>I put "answer" in quotes because it's ugly solution and because
there is no POSIX standard for pseudo-TTY devices (even though they
have a TTY standard...). What would make more sense to me would be to
have some way to set a mode on a file descriptor so that it will tell
the STDIO to be line-buffered. I have investigated, and I don't think
there is a way to set the buffered state of a child process. The STDIO
Library does not maintain any external state in the kernel or whatnot,
so I don't think there is any way for you to alter it. I'm not quite
sure how this line-buffered/block-buffered state change happens
internally in the STDIO library. I think the STDIO lib looks at the
file descriptor and decides to change behavior based on whether it's a
TTY or a block file (see isatty()).</p>
<p>I hope that this qualifies as helpful.</p>
<h1>Don't use a pipe to control another application...</h1>
<p>Pexpect may seem similar to <span class="code">os.popen()</span> or
<span class="code">commands</span> module. The main difference is that
Pexpect (like Expect) uses a pseudo-TTY to talk to the child
application. Most applications do no work well through the system()
call or through pipes. And probably all applications that ask a user to
type in a password will fail. These applications bypass the stdin and
read directly from the TTY device. Many applications do not explicitly
flush their output buffers. This causes deadlocks if you try to control
an interactive application using a pipe. What happens is that most UNIX
applications use the stdio (#include &lt;stdio.h&gt;) for input and
output. The stdio library behaves differently depending on where the
output is going. There is no way to control this behavior from the
client end.<br>
</p>
<p><b>Q: Can I do screen scraping with this thing?</b></p>
<p>A: That depends. If your application just does line-oriented output
then this is easy. If it does screen-oriented output then it may work,
but it could be hard. For example, trying to scrape data from the 'top'
command would be hard. The top command repaints the text window. </p>
<p>I am working on an ANSI / VT100 terminal emulator that will have
methods to get characters from an arbitrary X,Y coordinate of the
virtual screen. It works and you can play with it, but I have no
working examples at this time.</p>
<hr noshade="noshade" size="1">
<h1><a name="bugs"></a>Bugs</h1>
<h2>Threads</h2>
<p>On Linux (RH 8) you cannot spawn a child from a different thread and
pass the handle back to a worker thread. The child is successfully
spawned but you can't interact with it. The only way to make it work is
to spawn and interact with the child all in the same thread. [Adam
Kerrison] </p>
<h2><a name="echo_bug"></a>Timing issue with send() and sendline()</h2>
<p>This problem has been addressed and should not effect most users.</p>
<p>It is sometimes possible to read an echo of the string sent with <span
class="code">send()</span> and <span class="code">sendline()</span>.
If you call <span class="code">sendline()</span> and then immediately
call <span class="code">readline()</span> you may get part of your
output echoed back. You may read back what you just wrote even if the
child application does not explicitly echo it. Timing is critical. This
could be a security issue when talking to an application that asks for
a password; otherwise, this does not seem like a big deal. <i>But why
do TTYs do this</i>?</p>
<p>People usually report this when they are trying to control SSH or
some other login. For example, if your code looks something like this: </p>
<pre class="code">child.expect ('[pP]assword:')<br>child.sendline (my_password)</pre>
<p><br>
<blockquote>
1. SSH prints "password:" prompt to the user.<br>
2. SSH turns off echo on the TTY device.<br>
3. SSH waits for user to enter a password.<br>
</blockquote>
When scripting with Pexpect what can happen is that Pexpect will response to the "password:" prompt
before SSH has had time to turn off TTY echo. In other words, Pexpect sends the password between
steps 1. and 2., so the password gets echoed back to the TTY. I would call this an SSH bug.
</p>
<p>
Pexpect now automatically adds a short delay before sending data to a child process.
This more closely mimics what happens in the usual human-to-app interaction.
The delay can be tuned with the 'delaybeforesend' attribute of the spawn class.
In general, this fixes the problem for everyone and so this should not be an issue
for most users. For some applications you might with to turn it off.
child = pexpect.spawn ("ssh user@example.com")
child.delaybeforesend = 0
</p>
<p><br>
</p>
<p>Try changing it to look like the following. I know that this fix
does not look correct, but it works. I have not figured out exactly
what is happening. You would think that the sleep should be after the
sendline(). The fact that the sleep helps when it's between the
expect() and the sendline() must be a clue.</p>
<pre class="code">child.expect ('[pP]assword:')<br>child.sendline (my_password)</pre>
<h2>Timing issue with isalive()</h2>
<p>Reading the state of isalive() immediately after a child exits may
sometimes return 1. This is a race condition. The child has closed its
file descriptor, but has not yet fully exited before Pexpect's
isalive() executes. Addings a slight delay before the isalive() will
help. In the following example <span class="code">isalive()</span>
sometimes returns 1:</p>
<blockquote>
<pre class="code">child = pexpect.spawn('ls')<br>child.expect(pexpect.EOF)<br>print child.isalive()</pre>
</blockquote>
<p>But if there is any delay before the call to <span class="code">isalive()</span>
then it will always return 0 as expected.</p>
<blockquote>
<pre class="code">child = pexpect.spawn('ls')<br>child.expect(pexpect.EOF)<br>time.sleep(0.1)<br>print child.isalive()</pre>
</blockquote>
<h2>Truncated output just before child exits</h2>
<p><i>So far I have seen this only on older versions of <b>Apple's MacOS X</b>.</i>
If the child application quits it may not flush its output buffer. This
means that your Pexpect application will receive an EOF even though it
should have received a little more data before the child died. This is
not generally a problem when talking to interactive child applications.
One example where it is a problem is when trying to read output from a
program like '<span class="code">ls</span>'. You may receive most of
the directory listing, but the last few lines will get lost before you
receive an EOF. The reason for this is that '<span class="code">ls</span>'
runs; completes its task; and then exits. The buffer is not flushed
before exit so the last few lines are lost. The following example
demonstrates the problem:</p>
<p> </p>
<blockquote>
<pre class="code">child = pexpect.spawn ('ls -l')<br>child.expect (pexpect.EOF)<br>print child.before <br> </pre>
</blockquote>
<p></p>
<h2>Controlling SSH on Solaris</h2>
<p>Pexpect does not yet work perfectly on Solaris.
One common problem is that SSH sometimes will not allow TTY password
authentication. For example, you may expect SSH to ask you for a
password using code like this:
</p>
<pre class="code">child = pexpect.spawn ('ssh user@example.com')<br>child.expect ('assword')<br>child.sendline ('mypassword')<br></pre>
You may see the following error come back from a spawned
child SSH:
<p></p>
<blockquote>Permission denied (publickey,keyboard-interactive). </blockquote>
<p>
This means that SSH thinks it can't access the TTY to ask you for your
password.
The only solution I have found is to use public key authentication with
SSH.
This bypasses the need for a password. I'm not happy with this
solution.
The problem is due to poor support for Solaris Pseudo TTYs in the
Python
Standard Library. </p>
<hr noshade="noshade" size="1">
<h1><a name="changes"></a>CHANGES</h1>
<h2>Current Release</h2>
<p>Fixed OSError exception when a pexpect object is cleaned up.
Previously you might have seen this exception:</p>
<blockquote>
<pre class="code">Exception exceptions.OSError: (10, 'No child processes') <br>in &lt;bound method spawn.__del__ of<br>&lt;pexpect.spawn instance at 0xd248c&gt;&gt; ignored</pre>
</blockquote>
<p>You should not see that anymore. Thanks to Michael Surette.</p>
<p>Added support for buffering reads. This greatly improves speed when
trying to match long output from a child process. When you create an
instance of the spawn object you can then set a buffer size. For now
you MUST do the following to turn on buffering -- it may be on by
default in future version.</p>
<blockquote>
<pre class="code">child = pexpect.spawn ('my_command')<br>child.maxread=1000 # Sets buffer to 1000 characters.</pre>
</blockquote>
<div>
<p>I made a subtle change to the way TIMEOUT and EOF exceptions behave.
Previously you could either expect these states in which case pexpect
will not raise an exception, or you could just let pexpect raise an
exception when these states were encountered. If you expected the
states then the 'before' property was set to everything before the
state was encountered, but if you let pexpect raise the exception then
'before' was not set. Now the 'before' property will get set either way
you choose to handle these states.</p>
<h2><i>Older changes...</i></h2>
<p>The spawn object now provides iterators for a <i>file-like interface</i>.
This makes Pexpect a more complete file-like object. You can now write
code like this:</p>
<blockquote>
<pre class="code">child = pexpect.spawn ('ls -l')<br>for line in child:<br> print line<br></pre>
</blockquote>
<p>I added the attribute <span class="code">exitstatus</span>. This
will give the exit code returned by the child process. This will be set
to <span class="code">None</span> while the child is still alive. When
<span class="code">isalive()</span> returns 0 then <span class="code">exitstatus</span>
will be set.</p>
<p>I made a few more tweaks to <span class="code">isalive()</span> so
that it will operate more consistently on different platforms. Solaris
is the most difficult to support.</p>
<p>&nbsp;</p>
<p>You can now put <span class="code">TIMEOUT</span> in a list of
expected patterns. This is just like putting <span class="code">EOF</span>
in the pattern list. Expecting for a <span class="code">TIMEOUT</span>
may not be used as often as <span class="code">EOF</span>, but this
makes Pexpect more consitent.</p>
<p>Thanks to a suggestion and sample code from Chad J. Schroeder I
added the ability for Pexpect to operate on a file descriptor that is
already open. This means that Pexpect can be used to control streams
such as those from serial port devices. Now you just pass the integer
file descriptor as the "command" when contsructing a spawn open. For
example on a Linux box with a modem on ttyS1:</p>
<blockquote>
<pre class="code">fd = os.open("/dev/ttyS1", os.O_RDWR|os.O_NONBLOCK|os.O_NOCTTY)<br>m = pexpect.spawn(fd) # Note integer fd is used instead of usual string.<br>m.send("+++") # Escape sequence<br>m.send("ATZ0\r") # Reset modem to profile 0<br>rval = m.expect(["OK", "ERROR"])</pre>
</blockquote>
<h3>Pexpect now tests itself on Compile Farm!</h3>
<p>I wrote a nice script that uses ssh to connect to each machine on
Source Forge's Compile Farm and then run the testall.py script for each
platform. The result of the test is then recorded for each platform.
Now it's easy to run regression tests across multiple platforms.</p>
<h3>Pexpect is a file-like object</h3>
<p>The spawn object now provides a <i>file-like interface</i>. It
supports most of the methods and attributes defined for Python File
Objects. </p>
<p>I changed write and writelines() so that they no longer return a
value. Use send() if you need that functionality. I did this to make
the Spawn object more closely match a file-like object.</p>
<p>read() was renamed to read_nonblocking(). I added a new read()
method that matches file-like object interface. In general, you should
not notice the difference except that read() no longer allows you to
directly set the timeout value. I hope this will not effect any
existing code. Switching to read_nonblocking() should fix existing code.</p>
<p>I changed the name of <span class="code">set_echo()</span> to <span
class="code">setecho()</span>.</p>
<p>I changed the name of <span class="code">send_eof()</span> to <span
class="code">sendeof()</span>.</p>
<p>I modified <span class="code">kill()</span> so that it checks to
make sure the pid isalive().</p>
<p>I modified <span class="code">spawn()</span> (really called from <span
class="code">__spawn()</span>)so that it does not raise an expection
if <span class="code">setwinsize()</span> fails. Some platforms such
as Cygwin do not like setwinsize. This was a constant problem and since
it is not a critical feature I decided to just silence the error.
Normally I don't like to do that, but in this case I'm making an
exception.</p>
<p>Added a method <span class="code">close()</span> that does what you
think. It closes the file descriptor of the child application. It makes
no attempt to actually kill the child or wait for its status. </p>
<p>Add variables <span class="code">__version__</span> and <span
class="code">__revision__</span> (from cvs) to the pexpect modules.
This is mainly helpful to me so that I can make sure that I'm testing
with the right version instead of one already installed.</p>
<h3>Logging changes</h3>
<blockquote>
<p><span class="code">log_open()</span> and <span class="code">log_close()</span>
have been removed. Now use <span class="code">setlog()</span>. The <span
class="code">setlog()</span> method takes a file object. This is far
more flexible than the previous log method. Each time data is written
to the file object it will be flushed. To turn logging off simply call <span
class="code">setlog()</span> with None.</p>
</blockquote>
<h2>isalive changes</h2>
<blockquote>
<p>I renamed the <span class="code">isAlive()</span> method to <span
class="code">isalive()</span> to match the more typical naming style
in Python. Also the technique used to detect child process status has
been drastically modified. Previously I did some funky stuff with
signals which caused indigestion in other Python modules on some
platforms. It's was a big headache. It still is, but I think it works
better now.</p>
</blockquote>
<h3>attribute name changes</h3>
<blockquote>
<p>The names of some attributes have been changed. This effects the
names of the attributes that are set after called the <span
class="code">expect()</span> method.</p>
<table class="pymenu" border="0" cellpadding="5">
<tbody>
<tr>
<th class="pymenu">NEW NAME</th>
<th class="pymenu">OLD NAME</th>
</tr>
<tr>
<td><span class="code">before</span><br>
<i>Everything before the match.</i></td>
<td><span class="code">before</span></td>
</tr>
<tr>
<td><span class="code">after</span><br>
<i>Everything after and including the first character of the
match</i></td>
<td><span class="code">matched</span></td>
</tr>
<tr>
<td><span class="code">match</span><br>
<i>This is the re MatchObject from the match.<br>
You can get groups() from this.<br>
See '<span class="code">uptime.py</span>' in the examples tar ball.</i></td>
<td><i>New -- Did not exist</i></td>
</tr>
</tbody>
</table>
</blockquote>
<h3>EOF changes</h3>
<blockquote>
<p>The <span class="code">expect_eof()</span> method is gone. You
can now simply use the <span class="code">expect()</span> method to
look for EOF.</p>
<p>Was:</p>
<blockquote>
<p><span class="code">p.expect_eof ()</span></p>
</blockquote>
<p>Now:</p>
<blockquote>
<p><span class="code">p.expect (pexpect.EOF)</span></p>
</blockquote>
</blockquote>
<hr noshade="noshade" size="1">
<h1><a name="testing"></a>TESTING</h1>
<p>The following platforms have been tested:</p>
<!--
<table class="pymenu" border="0" cellpadding="5">
<tbody>
<tr>
<th class="pymenu">PLATFORM</th>
<th class="pymenu">RESULTS</th>
</tr>
<tr>
<td>Linux 2.4.9-ac10-rmk2-np1-cerf2<br>
armv4l</td>
<td><b><i>all tests passed</i></b></td>
</tr>
<tr>
<td>Linux 2.4.18 #2<br>
sparc64</td>
<td><b><i>all tests passed</i></b></td>
</tr>
<tr>
<td>MacOS X Darwin Kernel Version 5.5<br>
powerpc</td>
<td>
<p>failed more than one test.</p>
<p>Generally Pexpect works on OS X, but the nature of the quirks
cause a many of the tests to fail. See <a href="#bugs">bugs</a>
(Incomplete Child Output). The problem is more than minor, but Pexpect
is still more than useful for most tasks. The problem is an edge case.</p>
</td>
</tr>
<tr>
<td>Linux 2.2.20<br>
alpha<br>
</td>
<td><b><i>all tests passed</i></b></td>
</tr>
<tr>
<td>Linux 2.4.18-5smp<br>
i686</td>
<td><b><i>all tests passed</i></b></td>
</tr>
<tr>
<td>OpenBSD 2.9 GENERIC#653<br>
i386</td>
<td><b><i>all tests passed</i></b></td>
</tr>
<tr>
<td>Solaris</td>
<td>
<p>failed <span class="code">test_destructor</span></p>
<p>Otherwise, this is working pretty well. The destructor problem
is minor. For some reason, the <i>second</i> time a pty file
descriptor is created and deleted it never gets returned for use. It
does not effect the first time or the third time or any time after
that. It's only the second time. This is weird... This could be a file
descriptor leak, or it could be some peculiarity of how Solaris
recycles them. I thought it was a UNIX requirement for the OS to give
you the lowest available filedescriptor number. In any case, this
should not be a problem unless you create hundreds of pexpect
instances... It may also be a pty module bug. </p>
</td>
</tr>
<tr>
<td>Windows XP Cygwin</td>
<td>failed <span class="code">test_destructor</span>. That it
works at all is amazing to me. Cygwin rules!</td>
</tr>
</tbody>
</table>
-->
<h1>&nbsp;</h1>
<h1><a name="todo">TO DO</a></h1>
<p>Add an option to add a delay after each expect() or before each
read()/readline() call to automatically avoid the <a href="#echo_bug">echo
bug</a>.</p>
<p>&nbsp;</p>
</div>
<hr noshade="noshade" size="1">
<table border="0">
<tbody>
<tr>
<td> <a href="http://www.noah.org/email/"><img src="email.png"
alt="Click to send email." border="0" height="16" width="100"></a> </td>
</tr>
</tbody>
</table>
</div>
<div id="Menu"><b>INDEX</b><br>
<hr noshade="noshade" size="1"> <a href="#license"
title="Python Software Foundation License">License</a><br>
<a href="#download" title="Download and setup instructions">Download</a><br>
<a href="#doc" title="Documentation and overview">Documentation</a><br>
<a href="#status" title="Project Status">Project Status</a><br>
<a href="#requirements" title="System requirements to use Pexpect">Requirements</a><br>
<a href="#overview" title="Overview of what Pexpect does">Overview</a><br>
<a href="#faq" title="FAQ">FAQ</a><br>
<a href="#bugs" title="Bugs and work-arounds">Known Bugs</a><br>
<a href="#changes" title="What's new with Pexpect">Recent Changes</a><br>
<a href="#testing" title="Test results on various platforms">Testing</a><br>
<a href="#todo" title="What to do next">To do</a><br>
<a href="http://pexpect.svn.sourceforge.net/viewvc/pexpect/trunk/pexpect/" title="browse SVN">Browse SVN</a><br>
<br>
<a href="http://sourceforge.net/projects/pexpect/"
title="The Pexpect project page on SourceForge.net"> <img
src="http://sourceforge.net/sflogo.php?group_id=59762&amp;type=5"
alt="The Pexpect project page on SourceForge.net" border="0"
height="31" width="105"> </a> </div>
</body>
</html>

View File

@ -0,0 +1,72 @@
This directory contains scripts that give examples of using Pexpect.
hive.py
This script creates SSH connections to a list of hosts that
you provide. Then you are given a command line prompt. Each
shell command that you enter is sent to all the hosts. The
response from each host is collected and printed. For example,
you could connect to a dozen different machines and reboot
them all at once.
script.py
This implements a command similar to the classic BSD "script" command.
This will start a subshell and log all input and output to a file.
This demonstrates the interact() method of Pexpect.
fix_cvs_files.py
This is for cleaning up binary files improperly added to
CVS. This script scans the given path to find binary files;
checks with CVS to see if the sticky options are set to -kb;
finally if sticky options are not -kb then uses 'cvs admin'
to set the -kb option.
ftp.py
This demonstrates an FTP "bookmark".
This connects to an ftp site; does a few ftp commands; and then gives the user
interactive control over the session. In this case the "bookmark" is to a
directory on the OpenBSD ftp server. It puts you in the i386 packages
directory. You can easily modify this for other sites.
This demonstrates the interact() method of Pexpect.
monitor.py
This runs a sequence of system status commands on a remote host using SSH.
It runs a simple system checks such as uptime and free to monitor
the state of the remote host.
passmass.py
This will login to a list of hosts and change the password of the
given user. This demonstrates scripting logins; although, you could
more easily do this using the pxssh subclass of Pexpect.
See also the "hive.py" example script for a more general example
of scripting a collection of servers.
python.py
This starts the python interpreter and prints the greeting message backwards.
It then gives the user interactive control of Python. It's pretty useless!
rippy.py
This is a wizard for mencoder. It greatly simplifies the process of
ripping a DVD to mpeg4 format (XviD, DivX). It can transcode from any
video file to another. It has options for resampling the audio stream;
removing interlace artifacts, fitting to a target file size, etc.
There are lots of options, but the process is simple and easy to use.
sshls.py
This lists a directory on a remote machine.
ssh_tunnel.py
This starts an SSH tunnel to a remote machine. It monitors the connection
and restarts the tunnel if it goes down.
uptime.py
This will run the uptime command and parse the output into python variables.
This demonstrates using a single regular expression to match the output
of a command and capturing different variable in match groups.
The regular expression takes into account a wide variety of different
formats for uptime output.
df.py
This collects filesystem capacity info using the 'df' command.
Tuples of filesystem name and percentage are stored in a list.
A simple report is printed. Filesystems over 95% capacity are highlighted.

View File

@ -0,0 +1,74 @@
#!/usr/bin/env python
"""This runs Apache Status on the remote host and returns the number of requests per second.
./astat.py [-s server_hostname] [-u username] [-p password]
-s : hostname of the remote server to login to.
-u : username to user for login.
-p : Password to user for login.
Example:
This will print information about the given host:
./astat.py -s www.example.com -u mylogin -p mypassword
"""
import os, sys, time, re, getopt, getpass
import traceback
import pexpect, pxssh
def exit_with_usage():
print globals()['__doc__']
os._exit(1)
def main():
######################################################################
## Parse the options, arguments, get ready, etc.
######################################################################
try:
optlist, args = getopt.getopt(sys.argv[1:], 'h?s:u:p:', ['help','h','?'])
except Exception, e:
print str(e)
exit_with_usage()
options = dict(optlist)
if len(args) > 1:
exit_with_usage()
if [elem for elem in options if elem in ['-h','--h','-?','--?','--help']]:
print "Help:"
exit_with_usage()
if '-s' in options:
hostname = options['-s']
else:
hostname = raw_input('hostname: ')
if '-u' in options:
username = options['-u']
else:
username = raw_input('username: ')
if '-p' in options:
password = options['-p']
else:
password = getpass.getpass('password: ')
#
# Login via SSH
#
p = pxssh.pxssh()
p.login(hostname, username, password)
p.sendline('apachectl status')
p.expect('([0-9]+\.[0-9]+)\s*requests/sec')
requests_per_second = p.match.groups()[0]
p.logout()
print requests_per_second
if __name__ == "__main__":
try:
main()
except Exception, e:
print str(e)
traceback.print_exc()
os._exit(1)

View File

@ -0,0 +1,38 @@
#!/usr/bin/env python
"""This is a very simple client for the backdoor daemon. This is intended more
for testing rather than normal use. See bd_serv.py """
import socket
import sys, time, select
def recv_wrapper(s):
r,w,e = select.select([s.fileno()],[],[], 2)
if not r:
return ''
#cols = int(s.recv(4))
#rows = int(s.recv(4))
cols = 80
rows = 24
packet_size = cols * rows * 2 # double it for good measure
return s.recv(packet_size)
#HOST = '' #'localhost' # The remote host
#PORT = 1664 # The same port as used by the server
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.connect(sys.argv[1])#(HOST, PORT))
time.sleep(1)
#s.setblocking(0)
#s.send('COMMAND' + '\x01' + sys.argv[1])
s.send(':sendline ' + sys.argv[2])
print recv_wrapper(s)
s.close()
sys.exit()
#while True:
# data = recv_wrapper(s)
# if data == '':
# break
# sys.stdout.write (data)
# sys.stdout.flush()
#s.close()

View File

@ -0,0 +1,316 @@
#!/usr/bin/env python
"""Back door shell server
This exposes an shell terminal on a socket.
--hostname : sets the remote host name to open an ssh connection to.
--username : sets the user name to login with
--password : (optional) sets the password to login with
--port : set the local port for the server to listen on
--watch : show the virtual screen after each client request
"""
# Having the password on the command line is not a good idea, but
# then this entire project is probably not the most security concious thing
# I've ever built. This should be considered an experimental tool -- at best.
import pxssh, pexpect, ANSI
import time, sys, os, getopt, getpass, traceback, threading, socket
def exit_with_usage(exit_code=1):
print globals()['__doc__']
os._exit(exit_code)
class roller (threading.Thread):
"""This runs a function in a loop in a thread."""
def __init__(self, interval, function, args=[], kwargs={}):
"""The interval parameter defines time between each call to the function.
"""
threading.Thread.__init__(self)
self.interval = interval
self.function = function
self.args = args
self.kwargs = kwargs
self.finished = threading.Event()
def cancel(self):
"""Stop the roller."""
self.finished.set()
def run(self):
while not self.finished.isSet():
# self.finished.wait(self.interval)
self.function(*self.args, **self.kwargs)
def endless_poll (child, prompt, screen, refresh_timeout=0.1):
"""This keeps the screen updated with the output of the child. This runs in
a separate thread. See roller(). """
#child.logfile_read = screen
try:
s = child.read_nonblocking(4000, 0.1)
screen.write(s)
except:
pass
#while True:
# #child.prompt (timeout=refresh_timeout)
# try:
# #child.read_nonblocking(1,timeout=refresh_timeout)
# child.read_nonblocking(4000, 0.1)
# except:
# pass
def daemonize (stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
'''This forks the current process into a daemon. Almost none of this is
necessary (or advisable) if your daemon is being started by inetd. In that
case, stdin, stdout and stderr are all set up for you to refer to the
network connection, and the fork()s and session manipulation should not be
done (to avoid confusing inetd). Only the chdir() and umask() steps remain
as useful.
References:
UNIX Programming FAQ
1.7 How do I get my program to act like a daemon?
http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
Advanced Programming in the Unix Environment
W. Richard Stevens, 1992, Addison-Wesley, ISBN 0-201-56317-7.
The stdin, stdout, and stderr arguments are file names that will be opened
and be used to replace the standard file descriptors in sys.stdin,
sys.stdout, and sys.stderr. These arguments are optional and default to
/dev/null. Note that stderr is opened unbuffered, so if it shares a file
with stdout then interleaved output may not appear in the order that you
expect. '''
# Do first fork.
try:
pid = os.fork()
if pid > 0:
sys.exit(0) # Exit first parent.
except OSError, e:
sys.stderr.write ("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror) )
sys.exit(1)
# Decouple from parent environment.
os.chdir("/")
os.umask(0)
os.setsid()
# Do second fork.
try:
pid = os.fork()
if pid > 0:
sys.exit(0) # Exit second parent.
except OSError, e:
sys.stderr.write ("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror) )
sys.exit(1)
# Now I am a daemon!
# Redirect standard file descriptors.
si = open(stdin, 'r')
so = open(stdout, 'a+')
se = open(stderr, 'a+', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
# I now return as the daemon
return 0
def add_cursor_blink (response, row, col):
i = (row-1) * 80 + col
return response[:i]+'<img src="http://www.noah.org/cursor.gif">'+response[i:]
def main ():
try:
optlist, args = getopt.getopt(sys.argv[1:], 'h?d', ['help','h','?', 'hostname=', 'username=', 'password=', 'port=', 'watch'])
except Exception, e:
print str(e)
exit_with_usage()
command_line_options = dict(optlist)
options = dict(optlist)
# There are a million ways to cry for help. These are but a few of them.
if [elem for elem in command_line_options if elem in ['-h','--h','-?','--?','--help']]:
exit_with_usage(0)
hostname = "127.0.0.1"
port = 1664
username = os.getenv('USER')
password = ""
daemon_mode = False
if '-d' in options:
daemon_mode = True
if '--watch' in options:
watch_mode = True
else:
watch_mode = False
if '--hostname' in options:
hostname = options['--hostname']
if '--port' in options:
port = int(options['--port'])
if '--username' in options:
username = options['--username']
print "Login for %s@%s:%s" % (username, hostname, port)
if '--password' in options:
password = options['--password']
else:
password = getpass.getpass('password: ')
if daemon_mode:
print "daemonizing server"
daemonize()
#daemonize('/dev/null','/tmp/daemon.log','/tmp/daemon.log')
sys.stdout.write ('server started with pid %d\n' % os.getpid() )
virtual_screen = ANSI.ANSI (24,80)
child = pxssh.pxssh()
child.login (hostname, username, password)
print 'created shell. command line prompt is', child.PROMPT
#child.sendline ('stty -echo')
#child.setecho(False)
virtual_screen.write (child.before)
virtual_screen.write (child.after)
if os.path.exists("/tmp/mysock"): os.remove("/tmp/mysock")
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
localhost = '127.0.0.1'
s.bind('/tmp/mysock')
os.chmod('/tmp/mysock',0777)
print 'Listen'
s.listen(1)
print 'Accept'
#s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#localhost = '127.0.0.1'
#s.bind((localhost, port))
#print 'Listen'
#s.listen(1)
r = roller (0.01, endless_poll, (child, child.PROMPT, virtual_screen))
r.start()
print "screen poll updater started in background thread"
sys.stdout.flush()
try:
while True:
conn, addr = s.accept()
print 'Connected by', addr
data = conn.recv(1024)
if data[0]!=':':
cmd = ':sendline'
arg = data.strip()
else:
request = data.split(' ', 1)
if len(request)>1:
cmd = request[0].strip()
arg = request[1].strip()
else:
cmd = request[0].strip()
if cmd == ':exit':
r.cancel()
break
elif cmd == ':sendline':
child.sendline (arg)
#child.prompt(timeout=2)
time.sleep(0.2)
shell_window = str(virtual_screen)
elif cmd == ':send' or cmd==':xsend':
if cmd==':xsend':
arg = arg.decode("hex")
child.send (arg)
time.sleep(0.2)
shell_window = str(virtual_screen)
elif cmd == ':cursor':
shell_window = '%x%x' % (virtual_screen.cur_r, virtual_screen.cur_c)
elif cmd == ':refresh':
shell_window = str(virtual_screen)
response = []
response.append (shell_window)
#response = add_cursor_blink (response, row, col)
sent = conn.send('\n'.join(response))
if watch_mode: print '\n'.join(response)
if sent < len (response):
print "Sent is too short. Some data was cut off."
conn.close()
finally:
r.cancel()
print "cleaning up socket"
s.close()
if os.path.exists("/tmp/mysock"): os.remove("/tmp/mysock")
print "done!"
def pretty_box (rows, cols, s):
"""This puts an ASCII text box around the given string, s.
"""
top_bot = '+' + '-'*cols + '+\n'
return top_bot + '\n'.join(['|'+line+'|' for line in s.split('\n')]) + '\n' + top_bot
def error_response (msg):
response = []
response.append ("""All commands start with :
:{REQUEST} {ARGUMENT}
{REQUEST} may be one of the following:
:sendline: Run the ARGUMENT followed by a line feed.
:send : send the characters in the ARGUMENT without a line feed.
:refresh : Use to catch up the screen with the shell if state gets out of sync.
Example:
:sendline ls -l
You may also leave off :command and it will be assumed.
Example:
ls -l
is equivalent to:
:sendline ls -l
""")
response.append (msg)
return '\n'.join(response)
def parse_host_connect_string (hcs):
"""This parses a host connection string in the form
username:password@hostname:port. All fields are options expcet hostname. A
dictionary is returned with all four keys. Keys that were not included are
set to empty strings ''. Note that if your password has the '@' character
then you must backslash escape it. """
if '@' in hcs:
p = re.compile (r'(?P<username>[^@:]*)(:?)(?P<password>.*)(?!\\)@(?P<hostname>[^:]*):?(?P<port>[0-9]*)')
else:
p = re.compile (r'(?P<username>)(?P<password>)(?P<hostname>[^:]*):?(?P<port>[0-9]*)')
m = p.search (hcs)
d = m.groupdict()
d['password'] = d['password'].replace('\\@','@')
return d
if __name__ == "__main__":
try:
start_time = time.time()
print time.asctime()
main()
print time.asctime()
print "TOTAL TIME IN MINUTES:",
print (time.time() - start_time) / 60.0
except Exception, e:
print str(e)
tb_dump = traceback.format_exc()
print str(tb_dump)

View File

@ -0,0 +1,762 @@
#!/usr/bin/python
##!/usr/bin/env python
"""CGI shell server
This exposes a shell terminal on a web page.
It uses AJAX to send keys and receive screen updates.
The client web browser needs nothing but CSS and Javascript.
--hostname : sets the remote host name to open an ssh connection to.
--username : sets the user name to login with
--password : (optional) sets the password to login with
--port : set the local port for the server to listen on
--watch : show the virtual screen after each client request
This project is probably not the most security concious thing I've ever built.
This should be considered an experimental tool -- at best.
"""
import sys,os
sys.path.insert (0,os.getcwd()) # let local modules precede any installed modules
import socket, random, string, traceback, cgi, time, getopt, getpass, threading, resource, signal
import pxssh, pexpect, ANSI
def exit_with_usage(exit_code=1):
print globals()['__doc__']
os._exit(exit_code)
def client (command, host='localhost', port=-1):
"""This sends a request to the server and returns the response.
If port <= 0 then host is assumed to be the filename of a Unix domain socket.
If port > 0 then host is an inet hostname.
"""
if port <= 0:
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.connect(host)
else:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
s.send(command)
data = s.recv (2500)
s.close()
return data
def server (hostname, username, password, socket_filename='/tmp/server_sock', daemon_mode = True, verbose=False):
"""This starts and services requests from a client.
If daemon_mode is True then this forks off a separate daemon process and returns the daemon's pid.
If daemon_mode is False then this does not return until the server is done.
"""
if daemon_mode:
mypid_name = '/tmp/%d.pid' % os.getpid()
daemon_pid = daemonize(daemon_pid_filename=mypid_name)
time.sleep(1)
if daemon_pid != 0:
os.unlink(mypid_name)
return daemon_pid
virtual_screen = ANSI.ANSI (24,80)
child = pxssh.pxssh()
try:
child.login (hostname, username, password, login_naked=True)
except:
return
if verbose: print 'login OK'
virtual_screen.write (child.before)
virtual_screen.write (child.after)
if os.path.exists(socket_filename): os.remove(socket_filename)
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.bind(socket_filename)
os.chmod(socket_filename, 0777)
if verbose: print 'Listen'
s.listen(1)
r = roller (endless_poll, (child, child.PROMPT, virtual_screen))
r.start()
if verbose: print "started screen-poll-updater in background thread"
sys.stdout.flush()
try:
while True:
conn, addr = s.accept()
if verbose: print 'Connected by', addr
data = conn.recv(1024)
request = data.split(' ', 1)
if len(request)>1:
cmd = request[0].strip()
arg = request[1].strip()
else:
cmd = request[0].strip()
arg = ''
if cmd == 'exit':
r.cancel()
break
elif cmd == 'sendline':
child.sendline (arg)
time.sleep(0.1)
shell_window = str(virtual_screen)
elif cmd == 'send' or cmd=='xsend':
if cmd=='xsend':
arg = arg.decode("hex")
child.send (arg)
time.sleep(0.1)
shell_window = str(virtual_screen)
elif cmd == 'cursor':
shell_window = '%x,%x' % (virtual_screen.cur_r, virtual_screen.cur_c)
elif cmd == 'refresh':
shell_window = str(virtual_screen)
elif cmd == 'hash':
shell_window = str(hash(str(virtual_screen)))
response = []
response.append (shell_window)
if verbose: print '\n'.join(response)
sent = conn.send('\n'.join(response))
if sent < len (response):
if verbose: print "Sent is too short. Some data was cut off."
conn.close()
except e:
pass
r.cancel()
if verbose: print "cleaning up socket"
s.close()
if os.path.exists(socket_filename): os.remove(socket_filename)
if verbose: print "server done!"
class roller (threading.Thread):
"""This class continuously loops a function in a thread.
This is basically a thin layer around Thread with a
while loop and a cancel.
"""
def __init__(self, function, args=[], kwargs={}):
threading.Thread.__init__(self)
self.function = function
self.args = args
self.kwargs = kwargs
self.finished = threading.Event()
def cancel(self):
"""Stop the roller."""
self.finished.set()
def run(self):
while not self.finished.isSet():
self.function(*self.args, **self.kwargs)
def endless_poll (child, prompt, screen, refresh_timeout=0.1):
"""This keeps the screen updated with the output of the child.
This will be run in a separate thread. See roller class.
"""
#child.logfile_read = screen
try:
s = child.read_nonblocking(4000, 0.1)
screen.write(s)
except:
pass
def daemonize (stdin=None, stdout=None, stderr=None, daemon_pid_filename=None):
"""This runs the current process in the background as a daemon.
The arguments stdin, stdout, stderr allow you to set the filename that the daemon reads and writes to.
If they are set to None then all stdio for the daemon will be directed to /dev/null.
If daemon_pid_filename is set then the pid of the daemon will be written to it as plain text
and the pid will be returned. If daemon_pid_filename is None then this will return None.
"""
UMASK = 0
WORKINGDIR = "/"
MAXFD = 1024
# The stdio file descriptors are redirected to /dev/null by default.
if hasattr(os, "devnull"):
DEVNULL = os.devnull
else:
DEVNULL = "/dev/null"
if stdin is None: stdin = DEVNULL
if stdout is None: stdout = DEVNULL
if stderr is None: stderr = DEVNULL
try:
pid = os.fork()
except OSError, e:
raise Exception, "%s [%d]" % (e.strerror, e.errno)
if pid != 0: # The first child.
os.waitpid(pid,0)
if daemon_pid_filename is not None:
daemon_pid = int(file(daemon_pid_filename,'r').read())
return daemon_pid
else:
return None
# first child
os.setsid()
signal.signal(signal.SIGHUP, signal.SIG_IGN)
try:
pid = os.fork() # fork second child
except OSError, e:
raise Exception, "%s [%d]" % (e.strerror, e.errno)
if pid != 0:
if daemon_pid_filename is not None:
file(daemon_pid_filename,'w').write(str(pid))
os._exit(0) # exit parent (the first child) of the second child.
# second child
os.chdir(WORKINGDIR)
os.umask(UMASK)
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if maxfd == resource.RLIM_INFINITY:
maxfd = MAXFD
# close all file descriptors
for fd in xrange(0, maxfd):
try:
os.close(fd)
except OSError: # fd wasn't open to begin with (ignored)
pass
os.open (DEVNULL, os.O_RDWR) # standard input
# redirect standard file descriptors
si = open(stdin, 'r')
so = open(stdout, 'a+')
se = open(stderr, 'a+', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
return 0
def client_cgi ():
"""This handles the request if this script was called as a cgi.
"""
sys.stderr = sys.stdout
ajax_mode = False
TITLE="Shell"
SHELL_OUTPUT=""
SID="NOT"
print "Content-type: text/html;charset=utf-8\r\n"
try:
form = cgi.FieldStorage()
if form.has_key('ajax'):
ajax_mode = True
ajax_cmd = form['ajax'].value
SID=form['sid'].value
if ajax_cmd == 'send':
command = 'xsend'
arg = form['arg'].value.encode('hex')
result = client (command + ' ' + arg, '/tmp/'+SID)
print result
elif ajax_cmd == 'refresh':
command = 'refresh'
result = client (command, '/tmp/'+SID)
print result
elif ajax_cmd == 'cursor':
command = 'cursor'
result = client (command, '/tmp/'+SID)
print result
elif ajax_cmd == 'exit':
command = 'exit'
result = client (command, '/tmp/'+SID)
print result
elif ajax_cmd == 'hash':
command = 'hash'
result = client (command, '/tmp/'+SID)
print result
elif not form.has_key('sid'):
SID=random_sid()
print LOGIN_HTML % locals();
else:
SID=form['sid'].value
if form.has_key('start_server'):
USERNAME = form['username'].value
PASSWORD = form['password'].value
dpid = server ('127.0.0.1', USERNAME, PASSWORD, '/tmp/'+SID)
SHELL_OUTPUT="daemon pid: " + str(dpid)
else:
if form.has_key('cli'):
command = 'sendline ' + form['cli'].value
else:
command = 'sendline'
SHELL_OUTPUT = client (command, '/tmp/'+SID)
print CGISH_HTML % locals()
except:
tb_dump = traceback.format_exc()
if ajax_mode:
print str(tb_dump)
else:
SHELL_OUTPUT=str(tb_dump)
print CGISH_HTML % locals()
def server_cli():
"""This is the command line interface to starting the server.
This handles things if the script was not called as a CGI
(if you run it from the command line).
"""
try:
optlist, args = getopt.getopt(sys.argv[1:], 'h?d', ['help','h','?', 'hostname=', 'username=', 'password=', 'port=', 'watch'])
except Exception, e:
print str(e)
exit_with_usage()
command_line_options = dict(optlist)
options = dict(optlist)
# There are a million ways to cry for help. These are but a few of them.
if [elem for elem in command_line_options if elem in ['-h','--h','-?','--?','--help']]:
exit_with_usage(0)
hostname = "127.0.0.1"
#port = 1664
username = os.getenv('USER')
password = ""
daemon_mode = False
if '-d' in options:
daemon_mode = True
if '--watch' in options:
watch_mode = True
else:
watch_mode = False
if '--hostname' in options:
hostname = options['--hostname']
if '--port' in options:
port = int(options['--port'])
if '--username' in options:
username = options['--username']
if '--password' in options:
password = options['--password']
else:
password = getpass.getpass('password: ')
server (hostname, username, password, '/tmp/mysock', daemon_mode)
def random_sid ():
a=random.randint(0,65535)
b=random.randint(0,65535)
return '%04x%04x.sid' % (a,b)
def parse_host_connect_string (hcs):
"""This parses a host connection string in the form
username:password@hostname:port. All fields are options expcet hostname. A
dictionary is returned with all four keys. Keys that were not included are
set to empty strings ''. Note that if your password has the '@' character
then you must backslash escape it.
"""
if '@' in hcs:
p = re.compile (r'(?P<username>[^@:]*)(:?)(?P<password>.*)(?!\\)@(?P<hostname>[^:]*):?(?P<port>[0-9]*)')
else:
p = re.compile (r'(?P<username>)(?P<password>)(?P<hostname>[^:]*):?(?P<port>[0-9]*)')
m = p.search (hcs)
d = m.groupdict()
d['password'] = d['password'].replace('\\@','@')
return d
def pretty_box (s, rows=24, cols=80):
"""This puts an ASCII text box around the given string.
"""
top_bot = '+' + '-'*cols + '+\n'
return top_bot + '\n'.join(['|'+line+'|' for line in s.split('\n')]) + '\n' + top_bot
def main ():
if os.getenv('REQUEST_METHOD') is None:
server_cli()
else:
client_cgi()
# It's mostly HTML and Javascript from here on out.
CGISH_HTML="""<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<title>%(TITLE)s %(SID)s</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<style type=text/css>
a {color: #9f9; text-decoration: none}
a:hover {color: #0f0}
hr {color: #0f0}
html,body,textarea,input,form
{
font-family: "Courier New", Courier, mono;
font-size: 8pt;
color: #0c0;
background-color: #020;
margin:0;
padding:0;
border:0;
}
input { background-color: #010; }
textarea {
border-width:1;
border-style:solid;
border-color:#0c0;
padding:3;
margin:3;
}
</style>
<script language="JavaScript">
function focus_first()
{if (document.forms.length > 0)
{var TForm = document.forms[0];
for (i=0;i<TForm.length;i++){
if ((TForm.elements[i].type=="text")||
(TForm.elements[i].type=="textarea")||
(TForm.elements[i].type.toString().charAt(0)=="s"))
{document.forms[0].elements[i].focus();break;}}}}
// JavaScript Virtual Keyboard
// If you like this code then buy me a sandwich.
// Noah Spurrier <noah@noah.org>
var flag_shift=0;
var flag_shiftlock=0;
var flag_ctrl=0;
var ButtonOnColor="#ee0";
function init ()
{
// hack to set quote key to show both single quote and double quote
document.form['quote'].value = "'" + ' "';
//refresh_screen();
poll();
document.form["cli"].focus();
}
function get_password ()
{
var username = prompt("username?","");
var password = prompt("password?","");
start_server (username, password);
}
function multibrowser_ajax ()
{
var xmlHttp = false;
/*@cc_on @*/
/*@if (@_jscript_version >= 5)
try
{
xmlHttp = new ActiveXObject("Msxml2.XMLHTTP");
}
catch (e)
{
try
{
xmlHttp = new ActiveXObject("Microsoft.XMLHTTP");
}
catch (e2)
{
xmlHttp = false;
}
}
@end @*/
if (!xmlHttp && typeof XMLHttpRequest != 'undefined')
{
xmlHttp = new XMLHttpRequest();
}
return xmlHttp;
}
function load_url_to_screen(url)
{
xmlhttp = multibrowser_ajax();
//window.XMLHttpRequest?new XMLHttpRequest(): new ActiveXObject("Microsoft.XMLHTTP");
xmlhttp.onreadystatechange = update_virtual_screen;
xmlhttp.open("GET", url);
xmlhttp.setRequestHeader("If-Modified-Since", "Sat, 1 Jan 2000 00:00:00 GMT");
xmlhttp.send(null);
}
function update_virtual_screen()
{
if ((xmlhttp.readyState == 4) && (xmlhttp.status == 200))
{
var screen_text = xmlhttp.responseText;
document.form["screen_text"].value = screen_text;
//var json_data = json_parse(xmlhttp.responseText);
}
}
function poll()
{
refresh_screen();
timerID = setTimeout("poll()", 2000);
// clearTimeout(timerID);
}
//function start_server (username, password)
//{
// load_url_to_screen('cgishell.cgi?ajax=serverstart&username=' + escape(username) + '&password=' + escape(password);
//}
function refresh_screen()
{
load_url_to_screen('cgishell.cgi?ajax=refresh&sid=%(SID)s');
}
function query_hash()
{
load_url_to_screen('cgishell.cgi?ajax=hash&sid=%(SID)s');
}
function query_cursor()
{
load_url_to_screen('cgishell.cgi?ajax=cursor&sid=%(SID)s');
}
function exit_server()
{
load_url_to_screen('cgishell.cgi?ajax=exit&sid=%(SID)s');
}
function type_key (chars)
{
var ch = '?';
if (flag_shiftlock || flag_shift)
{
ch = chars.substr(1,1);
}
else if (flag_ctrl)
{
ch = chars.substr(2,1);
}
else
{
ch = chars.substr(0,1);
}
load_url_to_screen('cgishell.cgi?ajax=send&sid=%(SID)s&arg=' + escape(ch));
if (flag_shift || flag_ctrl)
{
flag_shift = 0;
flag_ctrl = 0;
}
update_button_colors();
}
function key_shiftlock()
{
flag_ctrl = 0;
flag_shift = 0;
if (flag_shiftlock)
{
flag_shiftlock = 0;
}
else
{
flag_shiftlock = 1;
}
update_button_colors();
}
function key_shift()
{
if (flag_shift)
{
flag_shift = 0;
}
else
{
flag_ctrl = 0;
flag_shiftlock = 0;
flag_shift = 1;
}
update_button_colors();
}
function key_ctrl ()
{
if (flag_ctrl)
{
flag_ctrl = 0;
}
else
{
flag_ctrl = 1;
flag_shiftlock = 0;
flag_shift = 0;
}
update_button_colors();
}
function update_button_colors ()
{
if (flag_ctrl)
{
document.form['Ctrl'].style.backgroundColor = ButtonOnColor;
document.form['Ctrl2'].style.backgroundColor = ButtonOnColor;
}
else
{
document.form['Ctrl'].style.backgroundColor = document.form.style.backgroundColor;
document.form['Ctrl2'].style.backgroundColor = document.form.style.backgroundColor;
}
if (flag_shift)
{
document.form['Shift'].style.backgroundColor = ButtonOnColor;
document.form['Shift2'].style.backgroundColor = ButtonOnColor;
}
else
{
document.form['Shift'].style.backgroundColor = document.form.style.backgroundColor;
document.form['Shift2'].style.backgroundColor = document.form.style.backgroundColor;
}
if (flag_shiftlock)
{
document.form['ShiftLock'].style.backgroundColor = ButtonOnColor;
}
else
{
document.form['ShiftLock'].style.backgroundColor = document.form.style.backgroundColor;
}
}
function keyHandler(e)
{
var pressedKey;
if (document.all) { e = window.event; }
if (document.layers) { pressedKey = e.which; }
if (document.all) { pressedKey = e.keyCode; }
pressedCharacter = String.fromCharCode(pressedKey);
type_key(pressedCharacter+pressedCharacter+pressedCharacter);
alert(pressedCharacter);
// alert(' Character = ' + pressedCharacter + ' [Decimal value = ' + pressedKey + ']');
}
//document.onkeypress = keyHandler;
//if (document.layers)
// document.captureEvents(Event.KEYPRESS);
//http://sniptools.com/jskeys
//document.onkeyup = KeyCheck;
function KeyCheck(e)
{
var KeyID = (window.event) ? event.keyCode : e.keyCode;
type_key(String.fromCharCode(KeyID));
e.cancelBubble = true;
window.event.cancelBubble = true;
}
</script>
</head>
<body onload="init()">
<form id="form" name="form" action="/cgi-bin/cgishell.cgi" method="POST">
<input name="sid" value="%(SID)s" type="hidden">
<textarea name="screen_text" cols="81" rows="25">%(SHELL_OUTPUT)s</textarea>
<hr noshade="1">
&nbsp;<input name="cli" id="cli" type="text" size="80"><br>
<table border="0" align="left">
<tr>
<td width="86%%" align="center">
<input name="submit" type="submit" value="Submit">
<input name="refresh" type="button" value="REFRESH" onclick="refresh_screen()">
<input name="refresh" type="button" value="CURSOR" onclick="query_cursor()">
<input name="hash" type="button" value="HASH" onclick="query_hash()">
<input name="exit" type="button" value="EXIT" onclick="exit_server()">
<br>
<input type="button" value="Esc" onclick="type_key('\\x1b\\x1b')" />
<input type="button" value="` ~" onclick="type_key('`~')" />
<input type="button" value="1!" onclick="type_key('1!')" />
<input type="button" value="2@" onclick="type_key('2@\\x00')" />
<input type="button" value="3#" onclick="type_key('3#')" />
<input type="button" value="4$" onclick="type_key('4$')" />
<input type="button" value="5%%" onclick="type_key('5%%')" />
<input type="button" value="6^" onclick="type_key('6^\\x1E')" />
<input type="button" value="7&" onclick="type_key('7&')" />
<input type="button" value="8*" onclick="type_key('8*')" />
<input type="button" value="9(" onclick="type_key('9(')" />
<input type="button" value="0)" onclick="type_key('0)')" />
<input type="button" value="-_" onclick="type_key('-_\\x1F')" />
<input type="button" value="=+" onclick="type_key('=+')" />
<input type="button" value="BkSp" onclick="type_key('\\x08\\x08\\x08')" />
<br>
<input type="button" value="Tab" onclick="type_key('\\t\\t')" />
<input type="button" value="Q" onclick="type_key('qQ\\x11')" />
<input type="button" value="W" onclick="type_key('wW\\x17')" />
<input type="button" value="E" onclick="type_key('eE\\x05')" />
<input type="button" value="R" onclick="type_key('rR\\x12')" />
<input type="button" value="T" onclick="type_key('tT\\x14')" />
<input type="button" value="Y" onclick="type_key('yY\\x19')" />
<input type="button" value="U" onclick="type_key('uU\\x15')" />
<input type="button" value="I" onclick="type_key('iI\\x09')" />
<input type="button" value="O" onclick="type_key('oO\\x0F')" />
<input type="button" value="P" onclick="type_key('pP\\x10')" />
<input type="button" value="[ {" onclick="type_key('[{\\x1b')" />
<input type="button" value="] }" onclick="type_key(']}\\x1d')" />
<input type="button" value="\\ |" onclick="type_key('\\\\|\\x1c')" />
<br>
<input type="button" id="Ctrl" value="Ctrl" onclick="key_ctrl()" />
<input type="button" value="A" onclick="type_key('aA\\x01')" />
<input type="button" value="S" onclick="type_key('sS\\x13')" />
<input type="button" value="D" onclick="type_key('dD\\x04')" />
<input type="button" value="F" onclick="type_key('fF\\x06')" />
<input type="button" value="G" onclick="type_key('gG\\x07')" />
<input type="button" value="H" onclick="type_key('hH\\x08')" />
<input type="button" value="J" onclick="type_key('jJ\\x0A')" />
<input type="button" value="K" onclick="type_key('kK\\x0B')" />
<input type="button" value="L" onclick="type_key('lL\\x0C')" />
<input type="button" value="; :" onclick="type_key(';:')" />
<input type="button" id="quote" value="'" onclick="type_key('\\x27\\x22')" />
<input type="button" value="Enter" onclick="type_key('\\n\\n')" />
<br>
<input type="button" id="ShiftLock" value="Caps Lock" onclick="key_shiftlock()" />
<input type="button" id="Shift" value="Shift" onclick="key_shift()" />
<input type="button" value="Z" onclick="type_key('zZ\\x1A')" />
<input type="button" value="X" onclick="type_key('xX\\x18')" />
<input type="button" value="C" onclick="type_key('cC\\x03')" />
<input type="button" value="V" onclick="type_key('vV\\x16')" />
<input type="button" value="B" onclick="type_key('bB\\x02')" />
<input type="button" value="N" onclick="type_key('nN\\x0E')" />
<input type="button" value="M" onclick="type_key('mM\\x0D')" />
<input type="button" value=", <" onclick="type_key(',<')" />
<input type="button" value=". >" onclick="type_key('.>')" />
<input type="button" value="/ ?" onclick="type_key('/?')" />
<input type="button" id="Shift2" value="Shift" onclick="key_shift()" />
<input type="button" id="Ctrl2" value="Ctrl" onclick="key_ctrl()" />
<br>
<input type="button" value=" FINAL FRONTIER " onclick="type_key(' ')" />
</td>
</tr>
</table>
</form>
</body>
</html>
"""
LOGIN_HTML="""<html>
<head>
<title>Shell Login</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<style type=text/css>
a {color: #9f9; text-decoration: none}
a:hover {color: #0f0}
hr {color: #0f0}
html,body,textarea,input,form
{
font-family: "Courier New", Courier, mono;
font-size: 8pt;
color: #0c0;
background-color: #020;
margin:3;
padding:0;
border:0;
}
input { background-color: #010; }
input,textarea {
border-width:1;
border-style:solid;
border-color:#0c0;
padding:3;
margin:3;
}
</style>
<script language="JavaScript">
function init ()
{
document.login_form["username"].focus();
}
</script>
</head>
<body onload="init()">
<form name="login_form" method="POST">
<input name="start_server" value="1" type="hidden">
<input name="sid" value="%(SID)s" type="hidden">
username: <input name="username" type="text" size="30"><br>
password: <input name="password" type="password" size="30"><br>
<input name="submit" type="submit" value="enter">
</form>
<br>
</body>
</html>
"""
if __name__ == "__main__":
try:
main()
except Exception, e:
print str(e)
tb_dump = traceback.format_exc()
print str(tb_dump)

View File

@ -0,0 +1,131 @@
#!/usr/bin/env python
'''This demonstrates controlling a screen oriented application (curses).
It starts two instances of gnuchess and then pits them against each other.
'''
import pexpect
import string
import ANSI
REGEX_MOVE = '(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)'
REGEX_MOVE_PART = '(?:[0-9]|\x1b\[C)(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)'
class Chess:
def __init__(self, engine = "/usr/local/bin/gnuchess -a -h 1"):
self.child = pexpect.spawn (engine)
self.term = ANSI.ANSI ()
self.child.expect ('Chess')
if self.child.after != 'Chess':
raise IOError, 'incompatible chess program'
self.term.process_list (self.before)
self.term.process_list (self.after)
self.last_computer_move = ''
def read_until_cursor (self, r,c)
while 1:
self.child.read(1, 60)
self.term.process (c)
if self.term.cur_r == r and self.term.cur_c == c:
return 1
def do_first_move (self, move):
self.child.expect ('Your move is')
self.child.sendline (move)
self.term.process_list (self.before)
self.term.process_list (self.after)
return move
def do_move (self, move):
read_until_cursor (19,60)
#self.child.expect ('\[19;60H')
self.child.sendline (move)
print 'do_move' move
return move
def get_first_computer_move (self):
self.child.expect ('My move is')
self.child.expect (REGEX_MOVE)
# print '', self.child.after
return self.child.after
def get_computer_move (self):
print 'Here'
i = self.child.expect (['\[17;59H', '\[17;58H'])
print i
if i == 0:
self.child.expect (REGEX_MOVE)
if len(self.child.after) < 4:
self.child.after = self.child.after + self.last_computer_move[3]
if i == 1:
self.child.expect (REGEX_MOVE_PART)
self.child.after = self.last_computer_move[0] + self.child.after
print '', self.child.after
self.last_computer_move = self.child.after
return self.child.after
def switch (self):
self.child.sendline ('switch')
def set_depth (self, depth):
self.child.sendline ('depth')
self.child.expect ('depth=')
self.child.sendline ('%d' % depth)
def quit(self):
self.child.sendline ('quit')
import sys, os
print 'Starting...'
white = Chess()
white.child.echo = 1
white.child.expect ('Your move is')
white.set_depth(2)
white.switch()
move_white = white.get_first_computer_move()
print 'first move white:', move_white
white.do_move ('e7e5')
move_white = white.get_computer_move()
print 'move white:', move_white
white.do_move ('f8c5')
move_white = white.get_computer_move()
print 'move white:', move_white
white.do_move ('b8a6')
move_white = white.get_computer_move()
print 'move white:', move_white
sys.exit(1)
black = Chess()
white = Chess()
white.child.expect ('Your move is')
white.switch()
move_white = white.get_first_computer_move()
print 'first move white:', move_white
black.do_first_move (move_white)
move_black = black.get_first_computer_move()
print 'first move black:', move_black
white.do_move (move_black)
done = 0
while not done:
move_white = white.get_computer_move()
print 'move white:', move_white
black.do_move (move_white)
move_black = black.get_computer_move()
print 'move black:', move_black
white.do_move (move_black)
print 'tail of loop'
g.quit()

View File

@ -0,0 +1,131 @@
#!/usr/bin/env python
'''This demonstrates controlling a screen oriented application (curses).
It starts two instances of gnuchess and then pits them against each other.
'''
import pexpect
import string
import ANSI
import sys, os, time
class Chess:
def __init__(self, engine = "/usr/local/bin/gnuchess -a -h 1"):
self.child = pexpect.spawn (engine)
self.term = ANSI.ANSI ()
#self.child.expect ('Chess')
#if self.child.after != 'Chess':
# raise IOError, 'incompatible chess program'
#self.term.process_list (self.child.before)
#self.term.process_list (self.child.after)
self.last_computer_move = ''
def read_until_cursor (self, r,c, e=0):
'''Eventually something like this should move into the screen class or
a subclass. Maybe a combination of pexpect and screen...
'''
fout = open ('log','a')
while self.term.cur_r != r or self.term.cur_c != c:
try:
k = self.child.read(1, 10)
except Exception, e:
print 'EXCEPTION, (r,c):(%d,%d)\n' %(self.term.cur_r, self.term.cur_c)
sys.stdout.flush()
self.term.process (k)
fout.write ('(r,c):(%d,%d)\n' %(self.term.cur_r, self.term.cur_c))
fout.flush()
if e:
sys.stdout.write (k)
sys.stdout.flush()
if self.term.cur_r == r and self.term.cur_c == c:
fout.close()
return 1
print 'DIDNT EVEN HIT.'
fout.close()
return 1
def expect_region (self):
'''This is another method that would be moved into the
screen class.
'''
pass
def do_scan (self):
fout = open ('log','a')
while 1:
c = self.child.read(1,10)
self.term.process (c)
fout.write ('(r,c):(%d,%d)\n' %(self.term.cur_r, self.term.cur_c))
fout.flush()
sys.stdout.write (c)
sys.stdout.flush()
def do_move (self, move, e = 0):
time.sleep(1)
self.read_until_cursor (19,60, e)
self.child.sendline (move)
def wait (self, color):
while 1:
r = self.term.get_region (14,50,14,60)[0]
r = r.strip()
if r == color:
return
time.sleep (1)
def parse_computer_move (self, s):
i = s.find ('is: ')
cm = s[i+3:i+9]
return cm
def get_computer_move (self, e = 0):
time.sleep(1)
self.read_until_cursor (19,60, e)
time.sleep(1)
r = self.term.get_region (17,50,17,62)[0]
cm = self.parse_computer_move (r)
return cm
def switch (self):
print 'switching'
self.child.sendline ('switch')
def set_depth (self, depth):
self.child.sendline ('depth')
self.child.expect ('depth=')
self.child.sendline ('%d' % depth)
def quit(self):
self.child.sendline ('quit')
def LOG (s):
print s
sys.stdout.flush ()
fout = open ('moves.log', 'a')
fout.write (s + '\n')
fout.close()
print 'Starting...'
black = Chess()
white = Chess()
white.read_until_cursor (19,60,1)
white.switch()
done = 0
while not done:
white.wait ('Black')
move_white = white.get_computer_move(1)
LOG ( 'move white:'+ move_white )
black.do_move (move_white)
black.wait ('White')
move_black = black.get_computer_move()
LOG ( 'move black:'+ move_black )
white.do_move (move_black, 1)
g.quit()

View File

@ -0,0 +1,138 @@
#!/usr/bin/env python
'''This demonstrates controlling a screen oriented application (curses).
It starts two instances of gnuchess and then pits them against each other.
'''
import pexpect
import string
import ANSI
REGEX_MOVE = '(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)'
REGEX_MOVE_PART = '(?:[0-9]|\x1b\[C)(?:[a-z]|\x1b\[C)(?:[0-9]|\x1b\[C)'
class Chess:
def __init__(self, engine = "/usr/local/bin/gnuchess -a -h 1"):
self.child = pexpect.spawn (engine)
self.term = ANSI.ANSI ()
# self.child.expect ('Chess')
# if self.child.after != 'Chess':
# raise IOError, 'incompatible chess program'
# self.term.process_list (self.before)
# self.term.process_list (self.after)
self.last_computer_move = ''
def read_until_cursor (self, r,c):
fout = open ('log','a')
while 1:
k = self.child.read(1, 10)
self.term.process (k)
fout.write ('(r,c):(%d,%d)\n' %(self.term.cur_r, self.term.cur_c))
fout.flush()
if self.term.cur_r == r and self.term.cur_c == c:
fout.close()
return 1
sys.stdout.write (k)
sys.stdout.flush()
def do_scan (self):
fout = open ('log','a')
while 1:
c = self.child.read(1,10)
self.term.process (c)
fout.write ('(r,c):(%d,%d)\n' %(self.term.cur_r, self.term.cur_c))
fout.flush()
sys.stdout.write (c)
sys.stdout.flush()
def do_move (self, move):
self.read_until_cursor (19,60)
self.child.sendline (move)
return move
def get_computer_move (self):
print 'Here'
i = self.child.expect (['\[17;59H', '\[17;58H'])
print i
if i == 0:
self.child.expect (REGEX_MOVE)
if len(self.child.after) < 4:
self.child.after = self.child.after + self.last_computer_move[3]
if i == 1:
self.child.expect (REGEX_MOVE_PART)
self.child.after = self.last_computer_move[0] + self.child.after
print '', self.child.after
self.last_computer_move = self.child.after
return self.child.after
def switch (self):
self.child.sendline ('switch')
def set_depth (self, depth):
self.child.sendline ('depth')
self.child.expect ('depth=')
self.child.sendline ('%d' % depth)
def quit(self):
self.child.sendline ('quit')
import sys, os
print 'Starting...'
white = Chess()
white.do_move('b2b4')
white.read_until_cursor (19,60)
c1 = white.term.get_abs(17,58)
c2 = white.term.get_abs(17,59)
c3 = white.term.get_abs(17,60)
c4 = white.term.get_abs(17,61)
fout = open ('log','a')
fout.write ('Computer:%s%s%s%s\n' %(c1,c2,c3,c4))
fout.close()
white.do_move('c2c4')
white.read_until_cursor (19,60)
c1 = white.term.get_abs(17,58)
c2 = white.term.get_abs(17,59)
c3 = white.term.get_abs(17,60)
c4 = white.term.get_abs(17,61)
fout = open ('log','a')
fout.write ('Computer:%s%s%s%s\n' %(c1,c2,c3,c4))
fout.close()
white.do_scan ()
#white.do_move ('b8a6')
#move_white = white.get_computer_move()
#print 'move white:', move_white
sys.exit(1)
black = Chess()
white = Chess()
white.child.expect ('Your move is')
white.switch()
move_white = white.get_first_computer_move()
print 'first move white:', move_white
black.do_first_move (move_white)
move_black = black.get_first_computer_move()
print 'first move black:', move_black
white.do_move (move_black)
done = 0
while not done:
move_white = white.get_computer_move()
print 'move white:', move_white
black.do_move (move_white)
move_black = black.get_computer_move()
print 'move black:', move_black
white.do_move (move_black)
print 'tail of loop'
g.quit()

View File

@ -0,0 +1,34 @@
#!/usr/bin/env python
"""This collects filesystem capacity info using the 'df' command. Tuples of
filesystem name and percentage are stored in a list. A simple report is
printed. Filesystems over 95% capacity are highlighted. Note that this does not
parse filesystem names after the first space, so names with spaces in them will
be truncated. This will produce ambiguous results for automount filesystems on
Apple OSX. """
import pexpect
child = pexpect.spawn ('df')
# parse 'df' output into a list.
pattern = "\n(\S+).*?([0-9]+)%"
filesystem_list = []
for dummy in range (0, 1000):
i = child.expect ([pattern, pexpect.EOF])
if i == 0:
filesystem_list.append (child.match.groups())
else:
break
# Print report
print
for m in filesystem_list:
s = "Filesystem %s is at %s%%" % (m[0], m[1])
# highlight filesystems over 95% capacity
if int(m[1]) > 95:
s = '! ' + s
else:
s = ' ' + s
print s

View File

@ -0,0 +1,95 @@
#!/usr/bin/env python
"""This is for cleaning up binary files improperly added to CVS. This script
scans the given path to find binary files; checks with CVS to see if the sticky
options are set to -kb; finally if sticky options are not -kb then uses 'cvs
admin' to set the -kb option.
This script ignores CVS directories, symbolic links, and files not known under
CVS control (cvs status is 'Unknown').
Run this on a CHECKED OUT module sandbox, not on the repository itself. After
if fixes the sticky options on any files you should manually do a 'cvs commit'
to accept the changes. Then be sure to have all users do a 'cvs up -A' to
update the Sticky Option status.
Noah Spurrier
20030426
"""
import os, sys, time
import pexpect
VERBOSE = 1
def is_binary (filename):
"""Assume that any file with a character where the 8th bit is set is
binary. """
fin = open(filename, 'rb')
wholething = fin.read()
fin.close()
for c in wholething:
if ord(c) & 0x80:
return 1
return 0
def is_kb_sticky (filename):
"""This checks if 'cvs status' reports '-kb' for Sticky options. If the
Sticky Option status is '-ks' then this returns 1. If the status is
'Unknown' then it returns 1. Otherwise 0 is returned. """
try:
s = pexpect.spawn ('cvs status %s' % filename)
i = s.expect (['Sticky Options:\s*(.*)\r\n', 'Status: Unknown'])
if i==1 and VERBOSE:
print 'File not part of CVS repository:', filename
return 1 # Pretend it's OK.
if s.match.group(1) == '-kb':
return 1
s = None
except:
print 'Something went wrong trying to run external cvs command.'
print ' cvs status %s' % filename
print 'The cvs command returned:'
print s.before
return 0
def cvs_admin_kb (filename):
"""This uses 'cvs admin' to set the '-kb' sticky option. """
s = pexpect.run ('cvs admin -kb %s' % filename)
# There is a timing issue. If I run 'cvs admin' too quickly
# cvs sometimes has trouble obtaining the directory lock.
time.sleep(1)
def walk_and_clean_cvs_binaries (arg, dirname, names):
"""This contains the logic for processing files. This is the os.path.walk
callback. This skips dirnames that end in CVS. """
if len(dirname)>3 and dirname[-3:]=='CVS':
return
for n in names:
fullpath = os.path.join (dirname, n)
if os.path.isdir(fullpath) or os.path.islink(fullpath):
continue
if is_binary(fullpath):
if not is_kb_sticky (fullpath):
if VERBOSE: print fullpath
cvs_admin_kb (fullpath)
def main ():
if len(sys.argv) == 1:
root = '.'
else:
root = sys.argv[1]
os.path.walk (root, walk_and_clean_cvs_binaries, None)
if __name__ == '__main__':
main ()

View File

@ -0,0 +1,47 @@
#!/usr/bin/env python
"""This demonstrates an FTP "bookmark". This connects to an ftp site; does a
few ftp stuff; and then gives the user interactive control over the session. In
this case the "bookmark" is to a directory on the OpenBSD ftp server. It puts
you in the i386 packages directory. You can easily modify this for other sites.
"""
import pexpect
import sys
child = pexpect.spawn('ftp ftp.openbsd.org')
child.expect('(?i)name .*: ')
child.sendline('anonymous')
child.expect('(?i)password')
child.sendline('pexpect@sourceforge.net')
child.expect('ftp> ')
child.sendline('cd /pub/OpenBSD/3.7/packages/i386')
child.expect('ftp> ')
child.sendline('bin')
child.expect('ftp> ')
child.sendline('prompt')
child.expect('ftp> ')
child.sendline('pwd')
child.expect('ftp> ')
print("Escape character is '^]'.\n")
sys.stdout.write (child.after)
sys.stdout.flush()
child.interact() # Escape character defaults to ^]
# At this point this script blocks until the user presses the escape character
# or until the child exits. The human user and the child should be talking
# to each other now.
# At this point the script is running again.
print 'Left interactve mode.'
# The rest is not strictly necessary. This just demonstrates a few functions.
# This makes sure the child is dead; although it would be killed when Python exits.
if child.isalive():
child.sendline('bye') # Try to ask ftp child to exit.
child.close()
# Print the final state of the child. Normally isalive() should be FALSE.
if child.isalive():
print 'Child did not exit gracefully.'
else:
print 'Child exited gracefully.'

View File

@ -0,0 +1,437 @@
#!/usr/bin/env python
"""hive -- Hive Shell
This lets you ssh to a group of servers and control them as if they were one.
Each command you enter is sent to each host in parallel. The response of each
host is collected and printed. In normal synchronous mode Hive will wait for
each host to return the shell command line prompt. The shell prompt is used to
sync output.
Example:
$ hive.py --sameuser --samepass host1.example.com host2.example.net
username: myusername
password:
connecting to host1.example.com - OK
connecting to host2.example.net - OK
targetting hosts: 192.168.1.104 192.168.1.107
CMD (? for help) > uptime
=======================================================================
host1.example.com
-----------------------------------------------------------------------
uptime
23:49:55 up 74 days, 5:14, 2 users, load average: 0.15, 0.05, 0.01
=======================================================================
host2.example.net
-----------------------------------------------------------------------
uptime
23:53:02 up 1 day, 13:36, 2 users, load average: 0.50, 0.40, 0.46
=======================================================================
Other Usage Examples:
1. You will be asked for your username and password for each host.
hive.py host1 host2 host3 ... hostN
2. You will be asked once for your username and password.
This will be used for each host.
hive.py --sameuser --samepass host1 host2 host3 ... hostN
3. Give a username and password on the command-line:
hive.py user1:pass2@host1 user2:pass2@host2 ... userN:passN@hostN
You can use an extended host notation to specify username, password, and host
instead of entering auth information interactively. Where you would enter a
host name use this format:
username:password@host
This assumes that ':' is not part of the password. If your password contains a
':' then you can use '\\:' to indicate a ':' and '\\\\' to indicate a single
'\\'. Remember that this information will appear in the process listing. Anyone
on your machine can see this auth information. This is not secure.
This is a crude script that begs to be multithreaded. But it serves its
purpose.
Noah Spurrier
$Id: hive.py 509 2008-01-05 21:27:47Z noah $
"""
# TODO add feature to support username:password@host combination
# TODO add feature to log each host output in separate file
import sys, os, re, optparse, traceback, types, time, getpass
import pexpect, pxssh
import readline, atexit
#histfile = os.path.join(os.environ["HOME"], ".hive_history")
#try:
# readline.read_history_file(histfile)
#except IOError:
# pass
#atexit.register(readline.write_history_file, histfile)
CMD_HELP="""Hive commands are preceded by a colon : (just think of vi).
:target name1 name2 name3 ...
set list of hosts to target commands
:target all
reset list of hosts to target all hosts in the hive.
:to name command
send a command line to the named host. This is similar to :target, but
sends only one command and does not change the list of targets for future
commands.
:sync
set mode to wait for shell prompts after commands are run. This is the
default. When Hive first logs into a host it sets a special shell prompt
pattern that it can later look for to synchronize output of the hosts. If
you 'su' to another user then it can upset the synchronization. If you need
to run something like 'su' then use the following pattern:
CMD (? for help) > :async
CMD (? for help) > sudo su - root
CMD (? for help) > :prompt
CMD (? for help) > :sync
:async
set mode to not expect command line prompts (see :sync). Afterwards
commands are send to target hosts, but their responses are not read back
until :sync is run. This is useful to run before commands that will not
return with the special shell prompt pattern that Hive uses to synchronize.
:refresh
refresh the display. This shows the last few lines of output from all hosts.
This is similar to resync, but does not expect the promt. This is useful
for seeing what hosts are doing during long running commands.
:resync
This is similar to :sync, but it does not change the mode. It looks for the
prompt and thus consumes all input from all targetted hosts.
:prompt
force each host to reset command line prompt to the special pattern used to
synchronize all the hosts. This is useful if you 'su' to a different user
where Hive would not know the prompt to match.
:send my text
This will send the 'my text' wihtout a line feed to the targetted hosts.
This output of the hosts is not automatically synchronized.
:control X
This will send the given control character to the targetted hosts.
For example, ":control c" will send ASCII 3.
:exit
This will exit the hive shell.
"""
def login (args, cli_username=None, cli_password=None):
# I have to keep a separate list of host names because Python dicts are not ordered.
# I want to keep the same order as in the args list.
host_names = []
hive_connect_info = {}
hive = {}
# build up the list of connection information (hostname, username, password, port)
for host_connect_string in args:
hcd = parse_host_connect_string (host_connect_string)
hostname = hcd['hostname']
port = hcd['port']
if port == '':
port = None
if len(hcd['username']) > 0:
username = hcd['username']
elif cli_username is not None:
username = cli_username
else:
username = raw_input('%s username: ' % hostname)
if len(hcd['password']) > 0:
password = hcd['password']
elif cli_password is not None:
password = cli_password
else:
password = getpass.getpass('%s password: ' % hostname)
host_names.append(hostname)
hive_connect_info[hostname] = (hostname, username, password, port)
# build up the list of hive connections using the connection information.
for hostname in host_names:
print 'connecting to', hostname
try:
fout = file("log_"+hostname, "w")
hive[hostname] = pxssh.pxssh()
hive[hostname].login(*hive_connect_info[hostname])
print hive[hostname].before
hive[hostname].logfile = fout
print '- OK'
except Exception, e:
print '- ERROR',
print str(e)
print 'Skipping', hostname
hive[hostname] = None
return host_names, hive
def main ():
global options, args, CMD_HELP
if options.sameuser:
cli_username = raw_input('username: ')
else:
cli_username = None
if options.samepass:
cli_password = getpass.getpass('password: ')
else:
cli_password = None
host_names, hive = login(args, cli_username, cli_password)
synchronous_mode = True
target_hostnames = host_names[:]
print 'targetting hosts:', ' '.join(target_hostnames)
while True:
cmd = raw_input('CMD (? for help) > ')
cmd = cmd.strip()
if cmd=='?' or cmd==':help' or cmd==':h':
print CMD_HELP
continue
elif cmd==':refresh':
refresh (hive, target_hostnames, timeout=0.5)
for hostname in target_hostnames:
if hive[hostname] is None:
print '/============================================================================='
print '| ' + hostname + ' is DEAD'
print '\\-----------------------------------------------------------------------------'
else:
print '/============================================================================='
print '| ' + hostname
print '\\-----------------------------------------------------------------------------'
print hive[hostname].before
print '=============================================================================='
continue
elif cmd==':resync':
resync (hive, target_hostnames, timeout=0.5)
for hostname in target_hostnames:
if hive[hostname] is None:
print '/============================================================================='
print '| ' + hostname + ' is DEAD'
print '\\-----------------------------------------------------------------------------'
else:
print '/============================================================================='
print '| ' + hostname
print '\\-----------------------------------------------------------------------------'
print hive[hostname].before
print '=============================================================================='
continue
elif cmd==':sync':
synchronous_mode = True
resync (hive, target_hostnames, timeout=0.5)
continue
elif cmd==':async':
synchronous_mode = False
continue
elif cmd==':prompt':
for hostname in target_hostnames:
try:
if hive[hostname] is not None:
hive[hostname].set_unique_prompt()
except Exception, e:
print "Had trouble communicating with %s, so removing it from the target list." % hostname
print str(e)
hive[hostname] = None
continue
elif cmd[:5] == ':send':
cmd, txt = cmd.split(None,1)
for hostname in target_hostnames:
try:
if hive[hostname] is not None:
hive[hostname].send(txt)
except Exception, e:
print "Had trouble communicating with %s, so removing it from the target list." % hostname
print str(e)
hive[hostname] = None
continue
elif cmd[:3] == ':to':
cmd, hostname, txt = cmd.split(None,2)
if hive[hostname] is None:
print '/============================================================================='
print '| ' + hostname + ' is DEAD'
print '\\-----------------------------------------------------------------------------'
continue
try:
hive[hostname].sendline (txt)
hive[hostname].prompt(timeout=2)
print '/============================================================================='
print '| ' + hostname
print '\\-----------------------------------------------------------------------------'
print hive[hostname].before
except Exception, e:
print "Had trouble communicating with %s, so removing it from the target list." % hostname
print str(e)
hive[hostname] = None
continue
elif cmd[:7] == ':expect':
cmd, pattern = cmd.split(None,1)
print 'looking for', pattern
try:
for hostname in target_hostnames:
if hive[hostname] is not None:
hive[hostname].expect(pattern)
print hive[hostname].before
except Exception, e:
print "Had trouble communicating with %s, so removing it from the target list." % hostname
print str(e)
hive[hostname] = None
continue
elif cmd[:7] == ':target':
target_hostnames = cmd.split()[1:]
if len(target_hostnames) == 0 or target_hostnames[0] == all:
target_hostnames = host_names[:]
print 'targetting hosts:', ' '.join(target_hostnames)
continue
elif cmd == ':exit' or cmd == ':q' or cmd == ':quit':
break
elif cmd[:8] == ':control' or cmd[:5] == ':ctrl' :
cmd, c = cmd.split(None,1)
if ord(c)-96 < 0 or ord(c)-96 > 255:
print '/============================================================================='
print '| Invalid character. Must be [a-zA-Z], @, [, ], \\, ^, _, or ?'
print '\\-----------------------------------------------------------------------------'
continue
for hostname in target_hostnames:
try:
if hive[hostname] is not None:
hive[hostname].sendcontrol(c)
except Exception, e:
print "Had trouble communicating with %s, so removing it from the target list." % hostname
print str(e)
hive[hostname] = None
continue
elif cmd == ':esc':
for hostname in target_hostnames:
if hive[hostname] is not None:
hive[hostname].send(chr(27))
continue
#
# Run the command on all targets in parallel
#
for hostname in target_hostnames:
try:
if hive[hostname] is not None:
hive[hostname].sendline (cmd)
except Exception, e:
print "Had trouble communicating with %s, so removing it from the target list." % hostname
print str(e)
hive[hostname] = None
#
# print the response for each targeted host.
#
if synchronous_mode:
for hostname in target_hostnames:
try:
if hive[hostname] is None:
print '/============================================================================='
print '| ' + hostname + ' is DEAD'
print '\\-----------------------------------------------------------------------------'
else:
hive[hostname].prompt(timeout=2)
print '/============================================================================='
print '| ' + hostname
print '\\-----------------------------------------------------------------------------'
print hive[hostname].before
except Exception, e:
print "Had trouble communicating with %s, so removing it from the target list." % hostname
print str(e)
hive[hostname] = None
print '=============================================================================='
def refresh (hive, hive_names, timeout=0.5):
"""This waits for the TIMEOUT on each host.
"""
# TODO This is ideal for threading.
for hostname in hive_names:
hive[hostname].expect([pexpect.TIMEOUT,pexpect.EOF],timeout=timeout)
def resync (hive, hive_names, timeout=2, max_attempts=5):
"""This waits for the shell prompt for each host in an effort to try to get
them all to the same state. The timeout is set low so that hosts that are
already at the prompt will not slow things down too much. If a prompt match
is made for a hosts then keep asking until it stops matching. This is a
best effort to consume all input if it printed more than one prompt. It's
kind of kludgy. Note that this will always introduce a delay equal to the
timeout for each machine. So for 10 machines with a 2 second delay you will
get AT LEAST a 20 second delay if not more. """
# TODO This is ideal for threading.
for hostname in hive_names:
for attempts in xrange(0, max_attempts):
if not hive[hostname].prompt(timeout=timeout):
break
def parse_host_connect_string (hcs):
"""This parses a host connection string in the form
username:password@hostname:port. All fields are options expcet hostname. A
dictionary is returned with all four keys. Keys that were not included are
set to empty strings ''. Note that if your password has the '@' character
then you must backslash escape it. """
if '@' in hcs:
p = re.compile (r'(?P<username>[^@:]*)(:?)(?P<password>.*)(?!\\)@(?P<hostname>[^:]*):?(?P<port>[0-9]*)')
else:
p = re.compile (r'(?P<username>)(?P<password>)(?P<hostname>[^:]*):?(?P<port>[0-9]*)')
m = p.search (hcs)
d = m.groupdict()
d['password'] = d['password'].replace('\\@','@')
return d
if __name__ == '__main__':
try:
start_time = time.time()
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), usage=globals()['__doc__'], version='$Id: hive.py 509 2008-01-05 21:27:47Z noah $',conflict_handler="resolve")
parser.add_option ('-v', '--verbose', action='store_true', default=False, help='verbose output')
parser.add_option ('--samepass', action='store_true', default=False, help='Use same password for each login.')
parser.add_option ('--sameuser', action='store_true', default=False, help='Use same username for each login.')
(options, args) = parser.parse_args()
if len(args) < 1:
parser.error ('missing argument')
if options.verbose: print time.asctime()
main()
if options.verbose: print time.asctime()
if options.verbose: print 'TOTAL TIME IN MINUTES:',
if options.verbose: print (time.time() - start_time) / 60.0
sys.exit(0)
except KeyboardInterrupt, e: # Ctrl-C
raise e
except SystemExit, e: # sys.exit()
raise e
except Exception, e:
print 'ERROR, UNEXPECTED EXCEPTION'
print str(e)
traceback.print_exc()
os._exit(1)

View File

@ -0,0 +1,208 @@
#!/usr/bin/env python
""" This runs a sequence of commands on a remote host using SSH. It runs a
simple system checks such as uptime and free to monitor the state of the remote
host.
./monitor.py [-s server_hostname] [-u username] [-p password]
-s : hostname of the remote server to login to.
-u : username to user for login.
-p : Password to user for login.
Example:
This will print information about the given host:
./monitor.py -s www.example.com -u mylogin -p mypassword
It works like this:
Login via SSH (This is the hardest part).
Run and parse 'uptime'.
Run 'iostat'.
Run 'vmstat'.
Run 'netstat'
Run 'free'.
Exit the remote host.
"""
import os, sys, time, re, getopt, getpass
import traceback
import pexpect
#
# Some constants.
#
COMMAND_PROMPT = '[#$] ' ### This is way too simple for industrial use -- we will change is ASAP.
TERMINAL_PROMPT = '(?i)terminal type\?'
TERMINAL_TYPE = 'vt100'
# This is the prompt we get if SSH does not have the remote host's public key stored in the cache.
SSH_NEWKEY = '(?i)are you sure you want to continue connecting'
def exit_with_usage():
print globals()['__doc__']
os._exit(1)
def main():
global COMMAND_PROMPT, TERMINAL_PROMPT, TERMINAL_TYPE, SSH_NEWKEY
######################################################################
## Parse the options, arguments, get ready, etc.
######################################################################
try:
optlist, args = getopt.getopt(sys.argv[1:], 'h?s:u:p:', ['help','h','?'])
except Exception, e:
print str(e)
exit_with_usage()
options = dict(optlist)
if len(args) > 1:
exit_with_usage()
if [elem for elem in options if elem in ['-h','--h','-?','--?','--help']]:
print "Help:"
exit_with_usage()
if '-s' in options:
host = options['-s']
else:
host = raw_input('hostname: ')
if '-u' in options:
user = options['-u']
else:
user = raw_input('username: ')
if '-p' in options:
password = options['-p']
else:
password = getpass.getpass('password: ')
#
# Login via SSH
#
child = pexpect.spawn('ssh -l %s %s'%(user, host))
i = child.expect([pexpect.TIMEOUT, SSH_NEWKEY, COMMAND_PROMPT, '(?i)password'])
if i == 0: # Timeout
print 'ERROR! could not login with SSH. Here is what SSH said:'
print child.before, child.after
print str(child)
sys.exit (1)
if i == 1: # In this case SSH does not have the public key cached.
child.sendline ('yes')
child.expect ('(?i)password')
if i == 2:
# This may happen if a public key was setup to automatically login.
# But beware, the COMMAND_PROMPT at this point is very trivial and
# could be fooled by some output in the MOTD or login message.
pass
if i == 3:
child.sendline(password)
# Now we are either at the command prompt or
# the login process is asking for our terminal type.
i = child.expect ([COMMAND_PROMPT, TERMINAL_PROMPT])
if i == 1:
child.sendline (TERMINAL_TYPE)
child.expect (COMMAND_PROMPT)
#
# Set command prompt to something more unique.
#
COMMAND_PROMPT = "\[PEXPECT\]\$ "
child.sendline ("PS1='[PEXPECT]\$ '") # In case of sh-style
i = child.expect ([pexpect.TIMEOUT, COMMAND_PROMPT], timeout=10)
if i == 0:
print "# Couldn't set sh-style prompt -- trying csh-style."
child.sendline ("set prompt='[PEXPECT]\$ '")
i = child.expect ([pexpect.TIMEOUT, COMMAND_PROMPT], timeout=10)
if i == 0:
print "Failed to set command prompt using sh or csh style."
print "Response was:"
print child.before
sys.exit (1)
# Now we should be at the command prompt and ready to run some commands.
print '---------------------------------------'
print 'Report of commands run on remote host.'
print '---------------------------------------'
# Run uname.
child.sendline ('uname -a')
child.expect (COMMAND_PROMPT)
print child.before
if 'linux' in child.before.lower():
LINUX_MODE = 1
else:
LINUX_MODE = 0
# Run and parse 'uptime'.
child.sendline ('uptime')
child.expect('up\s+(.*?),\s+([0-9]+) users?,\s+load averages?: ([0-9]+\.[0-9][0-9]),?\s+([0-9]+\.[0-9][0-9]),?\s+([0-9]+\.[0-9][0-9])')
duration, users, av1, av5, av15 = child.match.groups()
days = '0'
hours = '0'
mins = '0'
if 'day' in duration:
child.match = re.search('([0-9]+)\s+day',duration)
days = str(int(child.match.group(1)))
if ':' in duration:
child.match = re.search('([0-9]+):([0-9]+)',duration)
hours = str(int(child.match.group(1)))
mins = str(int(child.match.group(2)))
if 'min' in duration:
child.match = re.search('([0-9]+)\s+min',duration)
mins = str(int(child.match.group(1)))
print
print 'Uptime: %s days, %s users, %s (1 min), %s (5 min), %s (15 min)' % (
duration, users, av1, av5, av15)
child.expect (COMMAND_PROMPT)
# Run iostat.
child.sendline ('iostat')
child.expect (COMMAND_PROMPT)
print child.before
# Run vmstat.
child.sendline ('vmstat')
child.expect (COMMAND_PROMPT)
print child.before
# Run free.
if LINUX_MODE:
child.sendline ('free') # Linux systems only.
child.expect (COMMAND_PROMPT)
print child.before
# Run df.
child.sendline ('df')
child.expect (COMMAND_PROMPT)
print child.before
# Run lsof.
child.sendline ('lsof')
child.expect (COMMAND_PROMPT)
print child.before
# # Run netstat
# child.sendline ('netstat')
# child.expect (COMMAND_PROMPT)
# print child.before
# # Run MySQL show status.
# child.sendline ('mysql -p -e "SHOW STATUS;"')
# child.expect (PASSWORD_PROMPT_MYSQL)
# child.sendline (password_mysql)
# child.expect (COMMAND_PROMPT)
# print
# print child.before
# Now exit the remote host.
child.sendline ('exit')
index = child.expect([pexpect.EOF, "(?i)there are stopped jobs"])
if index==1:
child.sendline("exit")
child.expect(EOF)
if __name__ == "__main__":
try:
main()
except Exception, e:
print str(e)
traceback.print_exc()
os._exit(1)

View File

@ -0,0 +1,90 @@
#!/usr/bin/env python
"""Change passwords on the named machines. passmass host1 host2 host3 . . .
Note that login shell prompt on remote machine must end in # or $. """
import pexpect
import sys, getpass
USAGE = '''passmass host1 host2 host3 . . .'''
COMMAND_PROMPT = '[$#] '
TERMINAL_PROMPT = r'Terminal type\?'
TERMINAL_TYPE = 'vt100'
SSH_NEWKEY = r'Are you sure you want to continue connecting \(yes/no\)\?'
def login(host, user, password):
child = pexpect.spawn('ssh -l %s %s'%(user, host))
fout = file ("LOG.TXT","wb")
child.setlog (fout)
i = child.expect([pexpect.TIMEOUT, SSH_NEWKEY, '[Pp]assword: '])
if i == 0: # Timeout
print 'ERROR!'
print 'SSH could not login. Here is what SSH said:'
print child.before, child.after
sys.exit (1)
if i == 1: # SSH does not have the public key. Just accept it.
child.sendline ('yes')
child.expect ('[Pp]assword: ')
child.sendline(password)
# Now we are either at the command prompt or
# the login process is asking for our terminal type.
i = child.expect (['Permission denied', TERMINAL_PROMPT, COMMAND_PROMPT])
if i == 0:
print 'Permission denied on host:', host
sys.exit (1)
if i == 1:
child.sendline (TERMINAL_TYPE)
child.expect (COMMAND_PROMPT)
return child
# (current) UNIX password:
def change_password(child, user, oldpassword, newpassword):
child.sendline('passwd')
i = child.expect(['[Oo]ld [Pp]assword', '.current.*password', '[Nn]ew [Pp]assword'])
# Root does not require old password, so it gets to bypass the next step.
if i == 0 or i == 1:
child.sendline(oldpassword)
child.expect('[Nn]ew [Pp]assword')
child.sendline(newpassword)
i = child.expect(['[Nn]ew [Pp]assword', '[Rr]etype', '[Rr]e-enter'])
if i == 0:
print 'Host did not like new password. Here is what it said...'
print child.before
child.send (chr(3)) # Ctrl-C
child.sendline('') # This should tell remote passwd command to quit.
return
child.sendline(newpassword)
def main():
if len(sys.argv) <= 1:
print USAGE
return 1
user = raw_input('Username: ')
password = getpass.getpass('Current Password: ')
newpassword = getpass.getpass('New Password: ')
newpasswordconfirm = getpass.getpass('Confirm New Password: ')
if newpassword != newpasswordconfirm:
print 'New Passwords do not match.'
return 1
for host in sys.argv[1:]:
child = login(host, user, password)
if child == None:
print 'Could not login to host:', host
continue
print 'Changing password on host:', host
change_password(child, user, password, newpassword)
child.expect(COMMAND_PROMPT)
child.sendline('exit')
if __name__ == '__main__':
try:
main()
except pexpect.ExceptionPexpect, e:
print str(e)

View File

@ -0,0 +1,22 @@
#!/usr/bin/env python
"""This starts the python interpreter; captures the startup message; then gives
the user interactive control over the session. Why? For fun... """
# Don't do this unless you like being John Malkovich
# c = pexpect.spawn ('/usr/bin/env python ./python.py')
import pexpect
c = pexpect.spawn ('/usr/bin/env python')
c.expect ('>>>')
print 'And now for something completely different...'
f = lambda s:s and f(s[1:])+s[0] # Makes a function to reverse a string.
print f(c.before)
print 'Yes, it\'s python, but it\'s backwards.'
print
print 'Escape character is \'^]\'.'
print c.after,
c.interact()
c.kill(1)
print 'is alive:', c.isalive()

View File

@ -0,0 +1,993 @@
#!/usr/bin/env python
"""Rippy!
This script helps to convert video from one format to another.
This is useful for ripping DVD to mpeg4 video (XviD, DivX).
Features:
* automatic crop detection
* mp3 audio compression with resampling options
* automatic bitrate calculation based on desired target size
* optional interlace removal, b/w video optimization, video scaling
Run the script with no arguments to start with interactive prompts:
rippy.py
Run the script with the filename of a config to start automatic mode:
rippy.py rippy.conf
After Rippy is finished it saves the current configuation in a file called
'rippy.conf' in the local directoy. This can be used to rerun process using the
exact same settings by passing the filename of the conf file as an argument to
Rippy. Rippy will read the options from the file instead of asking you for
options interactively. So if you run rippy with 'dry_run=1' then you can run
the process again later using the 'rippy.conf' file. Don't forget to edit
'rippy.conf' to set 'dry_run=0'!
If you run rippy with 'dry_run' and 'verbose' true then the output generated is
valid command line commands. you could (in theory) cut-and-paste the commands
to a shell prompt. You will need to tweak some values such as crop area and bit
rate because these cannot be calculated in a dry run. This is useful if you
want to get an idea of what Rippy plans to do.
For all the trouble that Rippy goes through to calculate the best bitrate for a
desired target video size it sometimes fails to get it right. Sometimes the
final video size will differ more than you wanted from the desired size, but if
you are really motivated and have a lot of time on your hands then you can run
Rippy again with a manually calculated bitrate. After all compression is done
the first time Rippy will recalculate the bitrate to give you the nearly exact
bitrate that would have worked. You can then edit the 'rippy.conf' file; set
the video_bitrate with this revised bitrate; and then run Rippy all over again.
There is nothing like 4-pass video compression to get it right! Actually, this
could be done in three passes since I don't need to do the second pass
compression before I calculate the revised bitrate. I'm also considering an
enhancement where Rippy would compress ten spread out chunks, 1-minute in
length to estimate the bitrate.
Free, open source, and all that good stuff.
Rippy Copyright (c) 2006 Noah Spurrier
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
Noah Spurrier
$Id: rippy.py 517 2008-08-18 22:23:56Z noah $
"""
import sys, os, re, math, stat, getopt, traceback, types, time
import pexpect
__version__ = '1.2'
__revision__ = '$Revision: 11 $'
__all__ = ['main', __version__, __revision__]
GLOBAL_LOGFILE_NAME = "rippy_%d.log" % os.getpid()
GLOBAL_LOGFILE = open (GLOBAL_LOGFILE_NAME, "wb")
###############################################################################
# This giant section defines the prompts and defaults used in interactive mode.
###############################################################################
# Python dictionaries are unordered, so
# I have this list that maintains the order of the keys.
prompts_key_order = (
'verbose_flag',
'dry_run_flag',
'video_source_filename',
'video_chapter',
'video_final_filename',
'video_length',
'video_aspect_ratio',
'video_scale',
'video_encode_passes',
'video_codec',
'video_fourcc_override',
'video_bitrate',
'video_bitrate_overhead',
'video_target_size',
'video_deinterlace_flag',
'video_crop_area',
'video_gray_flag',
'subtitle_id',
'audio_id',
'audio_codec',
'audio_raw_filename',
'audio_volume_boost',
'audio_sample_rate',
'audio_bitrate',
#'audio_lowpass_filter',
'delete_tmp_files_flag'
)
#
# The 'prompts' dictionary holds all the messages shown to the user in
# interactive mode. The 'prompts' dictionary schema is defined as follows:
# prompt_key : ( default value, prompt string, help string, level of difficulty (0,1,2) )
#
prompts = {
'video_source_filename':("dvd://1", 'video source filename?', """This is the filename of the video that you want to convert from.
It can be any file that mencoder supports.
You can also choose a DVD device using the dvd://1 syntax.
Title 1 is usually the main title on a DVD.""",0),
'video_chapter':("none",'video chapter?',"""This is the chapter number. Usually disks such as TV series seasons will be divided into chapters. Maybe be set to none.""",0),
'video_final_filename':("video_final.avi", "video final filename?", """This is the name of the final video.""",0),
'audio_raw_filename':("audiodump.wav", "audio raw filename?", """This is the audio raw PCM filename. This is prior to compression.
Note that mplayer automatically names this audiodump.wav, so don't change this.""",1000),
#'audio_compressed_filename':("audiodump.mp3","Audio compressed filename?", """This is the name of the compressed audio that will be mixed
#into the final video. Normally you don't need to change this.""",2),
'video_length':("none","video length in seconds?","""This sets the length of the video in seconds. This is used to estimate the
bitrate for a target video file size. Set to 'calc' to have Rippy calculate
the length. Set to 'none' if you don't want rippy to estimate the bitrate --
you will have to manually specify bitrate.""",1),
'video_aspect_ratio':("calc","aspect ratio?","""This sets the aspect ratio of the video. Most DVDs are 16/9 or 4/3.""",1),
'video_scale':("none","video scale?","""This scales the video to the given output size. The default is to do no scaling.
You may type in a resolution such as 320x240 or you may use presets.
qntsc: 352x240 (NTSC quarter screen)
qpal: 352x288 (PAL quarter screen)
ntsc: 720x480 (standard NTSC)
pal: 720x576 (standard PAL)
sntsc: 640x480 (square pixel NTSC)
spal: 768x576 (square pixel PAL)""",1),
'video_codec':("mpeg4","video codec?","""This is the video compression to use. This is passed directly to mencoder, so
any format that it recognizes should work. For XviD or DivX use mpeg4.
Almost all MS Windows systems support wmv2 out of the box.
Some common codecs include:
mjpeg, h263, h263p, h264, mpeg4, msmpeg4, wmv1, wmv2, mpeg1video, mpeg2video, huffyuv, ffv1.
""",2),
'audio_codec':("mp3","audio codec?","""This is the audio compression to use. This is passed directly to mencoder, so
any format that it recognizes will work.
Some common codecs include:
mp3, mp2, aac, pcm
See mencoder manual for details.""",2),
'video_fourcc_override':("XVID","force fourcc code?","""This forces the fourcc codec to the given value. XVID is safest for Windows.
The following are common fourcc values:
FMP4 - This is the mencoder default. This is the "real" value.
XVID - used by Xvid (safest)
DX50 -
MP4S - Microsoft""",2),
'video_encode_passes':("1","number of encode passes?","""This sets how many passes to use to encode the video. You can choose 1 or 2.
Using two pases takes twice as long as one pass, but produces a better
quality video. I found that the improvement is not that impressive.""",1),
'verbose_flag':("Y","verbose output?","""This sets verbose output. If true then all commands and arguments are printed
before they are run. This is useful to see exactly how commands are run.""",1),
'dry_run_flag':("N","dry run?","""This sets 'dry run' mode. If true then commands are not run. This is useful
if you want to see what would the script would do.""",1),
'video_bitrate':("calc","video bitrate?","""This sets the video bitrate. This overrides video_target_size.
Set to 'calc' to automatically estimate the bitrate based on the
video final target size. If you set video_length to 'none' then
you will have to specify this video_bitrate.""",1),
'video_target_size':("737280000","video final target size?","""This sets the target video size that you want to end up with.
This is over-ridden by video_bitrate. In other words, if you specify
video_bitrate then video_target_size is ignored.
Due to the unpredictable nature of VBR compression the final video size
may not exactly match. The following are common CDR sizes:
180MB CDR (21 minutes) holds 193536000 bytes
550MB CDR (63 minutes) holds 580608000 bytes
650MB CDR (74 minutes) holds 681984000 bytes
700MB CDR (80 minutes) holds 737280000 bytes""",0),
'video_bitrate_overhead':("1.0","bitrate overhead factor?","""Adjust this value if you want to leave more room for
other files such as subtitle files.
If you specify video_bitrate then this value is ignored.""",2),
'video_crop_area':("detect","crop area?","""This sets the crop area to remove black bars from the top or sides of the video.
This helps save space. Set to 'detect' to automatically detect the crop area.
Set to 'none' to not crop the video. Normally you don't need to change this.""",1),
'video_deinterlace_flag':("N","is the video interlaced?","""This sets the deinterlace flag. If set then mencoder will be instructed
to filter out interlace artifacts (using '-vf pp=md').""",1),
'video_gray_flag':("N","is the video black and white (gray)?","""This improves output for black and white video.""",1),
'subtitle_id':("None","Subtitle ID stream?","""This selects the subtitle stream to extract from the source video.
Normally, 0 is the English subtitle stream for a DVD.
Subtitles IDs with higher numbers may be other languages.""",1),
'audio_id':("128","audio ID stream?","""This selects the audio stream to extract from the source video.
If your source is a VOB file (DVD) then stream IDs start at 128.
Normally, 128 is the main audio track for a DVD.
Tracks with higher numbers may be other language dubs or audio commentary.""",1),
'audio_sample_rate':("32000","audio sample rate (Hz) 48000, 44100, 32000, 24000, 12000","""This sets the rate at which the compressed audio will be resampled.
DVD audio is 48 kHz whereas music CDs use 44.1 kHz. The higher the sample rate
the more space the audio track will take. That will leave less space for video.
32 kHz is a good trade-off if you are trying to fit a video onto a CD.""",1),
'audio_bitrate':("96","audio bitrate (kbit/s) 192, 128, 96, 64?","""This sets the bitrate for MP3 audio compression.
The higher the bitrate the more space the audio track will take.
That will leave less space for video. Most people find music to be acceptable
at 128 kBitS. 96 kBitS is a good trade-off if you are trying to fit a video onto a CD.""",1),
'audio_volume_boost':("none","volume dB boost?","""Many DVDs have very low audio volume. This sets an audio volume boost in Decibels.
Values of 6 to 10 usually adjust quiet DVDs to a comfortable level.""",1),
#'audio_lowpass_filter':("16","audio lowpass filter (kHz)?","""This sets the low-pass filter for the audio.
#Normally this should be half of the audio sample rate.
#This improves audio compression and quality.
#Normally you don't need to change this.""",1),
'delete_tmp_files_flag':("N","delete temporary files when finished?","""If Y then %s, audio_raw_filename, and 'divx2pass.log' will be deleted at the end."""%GLOBAL_LOGFILE_NAME,1)
}
##############################################################################
# This is the important convert control function
##############################################################################
def convert (options):
"""This is the heart of it all -- this performs an end-to-end conversion of
a video from one format to another. It requires a dictionary of options.
The conversion process will also add some keys to the dictionary
such as length of the video and crop area. The dictionary is returned.
This options dictionary could be used again to repeat the convert process
(it is also saved to rippy.conf as text).
"""
if options['subtitle_id'] is not None:
print "# extract subtitles"
apply_smart (extract_subtitles, options)
else:
print "# do not extract subtitles."
# Optimization
# I really only need to calculate the exact video length if the user
# selected 'calc' for video_bitrate
# or
# selected 'detect' for video_crop_area.
if options['video_bitrate']=='calc' or options['video_crop_area']=='detect':
# As strange as it seems, the only reliable way to calculate the length
# of a video (in seconds) is to extract the raw, uncompressed PCM audio stream
# and then calculate the length of that. This is because MP4 video is VBR, so
# you cannot get exact time based on compressed size.
if options['video_length']=='calc':
print "# extract PCM raw audio to %s" % (options['audio_raw_filename'])
apply_smart (extract_audio, options)
options['video_length'] = apply_smart (get_length, options)
print "# Length of raw audio file : %d seconds (%0.2f minutes)" % (options['video_length'], float(options['video_length'])/60.0)
if options['video_bitrate']=='calc':
options['video_bitrate'] = options['video_bitrate_overhead'] * apply_smart (calc_video_bitrate, options)
print "# video bitrate : " + str(options['video_bitrate'])
if options['video_crop_area']=='detect':
options['video_crop_area'] = apply_smart (crop_detect, options)
print "# crop area : " + str(options['video_crop_area'])
print "# compression estimate"
print apply_smart (compression_estimate, options)
print "# compress video"
apply_smart (compress_video, options)
'audio_volume_boost',
print "# delete temporary files:",
if options['delete_tmp_files_flag']:
print "yes"
apply_smart (delete_tmp_files, options)
else:
print "no"
# Finish by saving options to rippy.conf and
# calclating if final_size is less than target_size.
o = ["# options used to create video\n"]
video_actual_size = get_filesize (options['video_final_filename'])
if options['video_target_size'] != 'none':
revised_bitrate = calculate_revised_bitrate (options['video_bitrate'], options['video_target_size'], video_actual_size)
o.append("# revised video_bitrate : %d\n" % revised_bitrate)
for k,v in options.iteritems():
o.append (" %30s : %s\n" % (k, v))
print '# '.join(o)
fout = open("rippy.conf","wb").write(''.join(o))
print "# final actual video size = %d" % video_actual_size
if options['video_target_size'] != 'none':
if video_actual_size > options['video_target_size']:
print "# FINAL VIDEO SIZE IS GREATER THAN DESIRED TARGET"
print "# final video size is %d bytes over target size" % (video_actual_size - options['video_target_size'])
else:
print "# final video size is %d bytes under target size" % (options['video_target_size'] - video_actual_size)
print "# If you want to run the entire compression process all over again"
print "# to get closer to the target video size then trying using a revised"
print "# video_bitrate of %d" % revised_bitrate
return options
##############################################################################
def exit_with_usage(exit_code=1):
print globals()['__doc__']
print 'version:', globals()['__version__']
sys.stdout.flush()
os._exit(exit_code)
def check_missing_requirements ():
"""This list of missing requirements (mencoder, mplayer, lame, and mkvmerge).
Returns None if all requirements are in the execution path.
"""
missing = []
if pexpect.which("mencoder") is None:
missing.append("mencoder")
if pexpect.which("mplayer") is None:
missing.append("mplayer")
cmd = "mencoder -oac help"
(command_output, exitstatus) = run(cmd)
ar = re.findall("(mp3lame)", command_output)
if len(ar)==0:
missing.append("Mencoder was not compiled with mp3lame support.")
#if pexpect.which("lame") is None:
# missing.append("lame")
#if pexpect.which("mkvmerge") is None:
# missing.append("mkvmerge")
if len(missing)==0:
return None
return missing
def input_option (message, default_value="", help=None, level=0, max_level=0):
"""This is a fancy raw_input function.
If the user enters '?' then the contents of help is printed.
The 'level' and 'max_level' are used to adjust which advanced options
are printed. 'max_level' is the level of options that the user wants
to see. 'level' is the level of difficulty for this particular option.
If this level is <= the max_level the user wants then the
message is printed and user input is allowed; otherwise, the
default value is returned automatically without user input.
"""
if default_value != '':
message = "%s [%s] " % (message, default_value)
if level > max_level:
return default_value
while 1:
user_input = raw_input (message)
if user_input=='?':
print help
elif user_input=='':
return default_value
else:
break
return user_input
def progress_callback (d=None):
"""This callback simply prints a dot to show activity.
This is used when running external commands with pexpect.run.
"""
sys.stdout.write (".")
sys.stdout.flush()
def run(cmd):
global GLOBAL_LOGFILE
print >>GLOBAL_LOGFILE, cmd
(command_output, exitstatus) = pexpect.run(cmd, events={pexpect.TIMEOUT:progress_callback}, timeout=5, withexitstatus=True, logfile=GLOBAL_LOGFILE)
if exitstatus != 0:
print "RUN FAILED. RETURNED EXIT STATUS:", exitstatus
print >>GLOBAL_LOGFILE, "RUN FAILED. RETURNED EXIT STATUS:", exitstatus
return (command_output, exitstatus)
def apply_smart (func, args):
"""This is similar to func(**args), but this won't complain about
extra keys in 'args'. This ignores keys in 'args' that are
not required by 'func'. This passes None to arguments that are
not defined in 'args'. That's fine for arguments with a default valeue, but
that's a bug for required arguments. I should probably raise a TypeError.
The func parameter can be a function reference or a string.
If it is a string then it is converted to a function reference.
"""
if type(func) is type(''):
if func in globals():
func = globals()[func]
else:
raise NameError("name '%s' is not defined" % func)
if hasattr(func,'im_func'): # Handle case when func is a class method.
func = func.im_func
argcount = func.func_code.co_argcount
required_args = dict([(k,args.get(k)) for k in func.func_code.co_varnames[:argcount]])
return func(**required_args)
def count_unique (items):
"""This takes a list and returns a sorted list of tuples with a count of each unique item in the list.
Example 1:
count_unique(['a','b','c','a','c','c','a','c','c'])
returns:
[(5,'c'), (3,'a'), (1,'b')]
Example 2 -- get the most frequent item in a list:
count_unique(['a','b','c','a','c','c','a','c','c'])[0][1]
returns:
'c'
"""
stats = {}
for i in items:
if i in stats:
stats[i] = stats[i] + 1
else:
stats[i] = 1
stats = [(v, k) for k, v in stats.items()]
stats.sort()
stats.reverse()
return stats
def calculate_revised_bitrate (video_bitrate, video_target_size, video_actual_size):
"""This calculates a revised video bitrate given the video_bitrate used,
the actual size that resulted, and the video_target_size.
This can be used if you want to compress the video all over again in an
attempt to get closer to the video_target_size.
"""
return int(math.floor(video_bitrate * (float(video_target_size) / float(video_actual_size))))
def get_aspect_ratio (video_source_filename):
"""This returns the aspect ratio of the original video.
This is usualy 1.78:1(16/9) or 1.33:1(4/3).
This function is very lenient. It basically guesses 16/9 whenever
it cannot figure out the aspect ratio.
"""
cmd = "mplayer '%s' -vo png -ao null -frames 1" % video_source_filename
(command_output, exitstatus) = run(cmd)
ar = re.findall("Movie-Aspect is ([0-9]+\.?[0-9]*:[0-9]+\.?[0-9]*)", command_output)
if len(ar)==0:
return '16/9'
if ar[0] == '1.78:1':
return '16/9'
if ar[0] == '1.33:1':
return '4/3'
return '16/9'
#idh = re.findall("ID_VIDEO_HEIGHT=([0-9]+)", command_output)
#if len(idw)==0 or len(idh)==0:
# print 'WARNING!'
# print 'Could not get aspect ration. Assuming 1.78:1 (16/9).'
# return 1.78
#return float(idw[0])/float(idh[0])
#ID_VIDEO_WIDTH=720
#ID_VIDEO_HEIGHT=480
#Movie-Aspect is 1.78:1 - prescaling to correct movie aspect.
def get_aid_list (video_source_filename):
"""This returns a list of audio ids in the source video file.
TODO: Also extract ID_AID_nnn_LANG to associate language. Not all DVDs include this.
"""
cmd = "mplayer '%s' -vo null -ao null -frames 0 -identify" % video_source_filename
(command_output, exitstatus) = run(cmd)
idl = re.findall("ID_AUDIO_ID=([0-9]+)", command_output)
idl.sort()
return idl
def get_sid_list (video_source_filename):
"""This returns a list of subtitle ids in the source video file.
TODO: Also extract ID_SID_nnn_LANG to associate language. Not all DVDs include this.
"""
cmd = "mplayer '%s' -vo null -ao null -frames 0 -identify" % video_source_filename
(command_output, exitstatus) = run(cmd)
idl = re.findall("ID_SUBTITLE_ID=([0-9]+)", command_output)
idl.sort()
return idl
def extract_audio (video_source_filename, audio_id=128, verbose_flag=0, dry_run_flag=0):
"""This extracts the given audio_id track as raw uncompressed PCM from the given source video.
Note that mplayer always saves this to audiodump.wav.
At this time there is no way to set the output audio name.
"""
#cmd = "mplayer %(video_source_filename)s -vc null -vo null -aid %(audio_id)s -ao pcm:fast -noframedrop" % locals()
cmd = "mplayer -quiet '%(video_source_filename)s' -vc dummy -vo null -aid %(audio_id)s -ao pcm:fast -noframedrop" % locals()
if verbose_flag: print cmd
if not dry_run_flag:
run(cmd)
print
def extract_subtitles (video_source_filename, subtitle_id=0, verbose_flag=0, dry_run_flag=0):
"""This extracts the given subtitle_id track as VOBSUB format from the given source video.
"""
cmd = "mencoder -quiet '%(video_source_filename)s' -o /dev/null -nosound -ovc copy -vobsubout subtitles -vobsuboutindex 0 -sid %(subtitle_id)s" % locals()
if verbose_flag: print cmd
if not dry_run_flag:
run(cmd)
print
def get_length (audio_raw_filename):
"""This attempts to get the length of the media file (length is time in seconds).
This should not be confused with size (in bytes) of the file data.
This is best used on a raw PCM AUDIO file because mplayer cannot get an accurate
time for many compressed video and audio formats -- notably MPEG4 and MP3.
Weird...
This returns -1 if it cannot get the length of the given file.
"""
cmd = "mplayer %s -vo null -ao null -frames 0 -identify" % audio_raw_filename
(command_output, exitstatus) = run(cmd)
idl = re.findall("ID_LENGTH=([0-9.]*)", command_output)
idl.sort()
if len(idl) != 1:
print "ERROR: cannot get length of raw audio file."
print "command_output of mplayer identify:"
print command_output
print "parsed command_output:"
print str(idl)
return -1
return float(idl[0])
def get_filesize (filename):
"""This returns the number of bytes a file takes on storage."""
return os.stat(filename)[stat.ST_SIZE]
def calc_video_bitrate (video_target_size, audio_bitrate, video_length, extra_space=0, dry_run_flag=0):
"""This gives an estimate of the video bitrate necessary to
fit the final target size. This will take into account room to
fit the audio and extra space if given (for container overhead or whatnot).
video_target_size is in bytes,
audio_bitrate is bits per second (96, 128, 256, etc.) ASSUMING CBR,
video_length is in seconds,
extra_space is in bytes.
a 180MB CDR (21 minutes) holds 193536000 bytes.
a 550MB CDR (63 minutes) holds 580608000 bytes.
a 650MB CDR (74 minutes) holds 681984000 bytes.
a 700MB CDR (80 minutes) holds 737280000 bytes.
"""
if dry_run_flag:
return -1
if extra_space is None: extra_space = 0
#audio_size = os.stat(audio_compressed_filename)[stat.ST_SIZE]
audio_size = (audio_bitrate * video_length * 1000) / 8.0
video_target_size = video_target_size - audio_size - extra_space
return (int)(calc_video_kbitrate (video_target_size, video_length))
def calc_video_kbitrate (target_size, length_secs):
"""Given a target byte size free for video data, this returns the bitrate in kBit/S.
For mencoder vbitrate 1 kBit = 1000 Bits -- not 1024 bits.
target_size = bitrate * 1000 * length_secs / 8
target_size = bitrate * 125 * length_secs
bitrate = target_size/(125*length_secs)
"""
return int(target_size / (125.0 * length_secs))
def crop_detect (video_source_filename, video_length, dry_run_flag=0):
"""This attempts to figure out the best crop for the given video file.
Basically it runs crop detect for 10 seconds on five different places in the video.
It picks the crop area that was most often detected.
"""
skip = int(video_length/9) # offset to skip (-ss option in mencoder)
sample_length = 10
cmd1 = "mencoder '%s' -quiet -ss %d -endpos %d -o /dev/null -nosound -ovc lavc -vf cropdetect" % (video_source_filename, skip, sample_length)
cmd2 = "mencoder '%s' -quiet -ss %d -endpos %d -o /dev/null -nosound -ovc lavc -vf cropdetect" % (video_source_filename, 2*skip, sample_length)
cmd3 = "mencoder '%s' -quiet -ss %d -endpos %d -o /dev/null -nosound -ovc lavc -vf cropdetect" % (video_source_filename, 4*skip, sample_length)
cmd4 = "mencoder '%s' -quiet -ss %d -endpos %d -o /dev/null -nosound -ovc lavc -vf cropdetect" % (video_source_filename, 6*skip, sample_length)
cmd5 = "mencoder '%s' -quiet -ss %d -endpos %d -o /dev/null -nosound -ovc lavc -vf cropdetect" % (video_source_filename, 8*skip, sample_length)
if dry_run_flag:
return "0:0:0:0"
(command_output1, exitstatus1) = run(cmd1)
(command_output2, exitstatus2) = run(cmd2)
(command_output3, exitstatus3) = run(cmd3)
(command_output4, exitstatus4) = run(cmd4)
(command_output5, exitstatus5) = run(cmd5)
idl = re.findall("-vf crop=([0-9]+:[0-9]+:[0-9]+:[0-9]+)", command_output1)
idl = idl + re.findall("-vf crop=([0-9]+:[0-9]+:[0-9]+:[0-9]+)", command_output2)
idl = idl + re.findall("-vf crop=([0-9]+:[0-9]+:[0-9]+:[0-9]+)", command_output3)
idl = idl + re.findall("-vf crop=([0-9]+:[0-9]+:[0-9]+:[0-9]+)", command_output4)
idl = idl + re.findall("-vf crop=([0-9]+:[0-9]+:[0-9]+:[0-9]+)", command_output5)
items_count = count_unique(idl)
return items_count[0][1]
def build_compression_command (video_source_filename, video_final_filename, video_target_size, audio_id=128, video_bitrate=1000, video_codec='mpeg4', audio_codec='mp3', video_fourcc_override='FMP4', video_gray_flag=0, video_crop_area=None, video_aspect_ratio='16/9', video_scale=None, video_encode_passes=2, video_deinterlace_flag=0, audio_volume_boost=None, audio_sample_rate=None, audio_bitrate=None, seek_skip=None, seek_length=None, video_chapter=None):
#Notes:For DVD, VCD, and SVCD use acodec=mp2 and vcodec=mpeg2video:
#mencoder movie.avi -o movie.VOB -ovc lavc -oac lavc -lavcopts acodec=mp2:abitrate=224:vcodec=mpeg2video:vbitrate=2000
#
# build video filter (-vf) argument
#
video_filter = ''
if video_crop_area and video_crop_area.lower()!='none':
video_filter = video_filter + 'crop=%s' % video_crop_area
if video_deinterlace_flag:
if video_filter != '':
video_filter = video_filter + ','
video_filter = video_filter + 'pp=md'
if video_scale and video_scale.lower()!='none':
if video_filter != '':
video_filter = video_filter + ','
video_filter = video_filter + 'scale=%s' % video_scale
# optional video rotation -- were you holding your camera sideways?
#if video_filter != '':
# video_filter = video_filter + ','
#video_filter = video_filter + 'rotate=2'
if video_filter != '':
video_filter = '-vf ' + video_filter
#
# build chapter argument
#
if video_chapter is not None:
chapter = '-chapter %d-%d' %(video_chapter,video_chapter)
else:
chapter = ''
# chapter = '-chapter 2-2'
#
# build audio_filter argument
#
audio_filter = ''
if audio_sample_rate:
if audio_filter != '':
audio_filter = audio_filter + ','
audio_filter = audio_filter + 'lavcresample=%s' % audio_sample_rate
if audio_volume_boost is not None:
if audio_filter != '':
audio_filter = audio_filter + ','
audio_filter = audio_filter + 'volume=%0.1f:1'%audio_volume_boost
if audio_filter != '':
audio_filter = '-af ' + audio_filter
#
#if audio_sample_rate:
# audio_filter = ('-srate %d ' % audio_sample_rate) + audio_filter
#
# build lavcopts argument
#
#lavcopts = '-lavcopts vcodec=%s:vbitrate=%d:mbd=2:aspect=%s:acodec=%s:abitrate=%d:vpass=1' % (video_codec,video_bitrate,audio_codec,audio_bitrate)
lavcopts = '-lavcopts vcodec=%(video_codec)s:vbitrate=%(video_bitrate)d:mbd=2:aspect=%(video_aspect_ratio)s:acodec=%(audio_codec)s:abitrate=%(audio_bitrate)d:vpass=1' % (locals())
if video_gray_flag:
lavcopts = lavcopts + ':gray'
seek_filter = ''
if seek_skip is not None:
seek_filter = '-ss %s' % (str(seek_skip))
if seek_length is not None:
seek_filter = seek_filter + ' -endpos %s' % (str(seek_length))
# cmd = "mencoder -quiet -info comment='Arkivist' '%(video_source_filename)s' %(seek_filter)s %(chapter)s -aid %(audio_id)s -o '%(video_final_filename)s' -ffourcc %(video_fourcc_override)s -ovc lavc -oac lavc %(lavcopts)s %(video_filter)s %(audio_filter)s" % locals()
cmd = "mencoder -quiet -info comment='Arkivist' '%(video_source_filename)s' %(seek_filter)s %(chapter)s -aid %(audio_id)s -o '%(video_final_filename)s' -ffourcc %(video_fourcc_override)s -ovc lavc -oac mp3lame %(lavcopts)s %(video_filter)s %(audio_filter)s" % locals()
return cmd
def compression_estimate (video_length, video_source_filename, video_final_filename, video_target_size, audio_id=128, video_bitrate=1000, video_codec='mpeg4', audio_codec='mp3', video_fourcc_override='FMP4', video_gray_flag=0, video_crop_area=None, video_aspect_ratio='16/9', video_scale=None, video_encode_passes=2, video_deinterlace_flag=0, audio_volume_boost=None, audio_sample_rate=None, audio_bitrate=None):
"""This attempts to figure out the best compression ratio for a given set of compression options.
"""
# TODO Need to account for AVI overhead.
skip = int(video_length/9) # offset to skip (-ss option in mencoder)
sample_length = 10
cmd1 = build_compression_command (video_source_filename, "compression_test_1.avi", video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, skip, sample_length)
cmd2 = build_compression_command (video_source_filename, "compression_test_2.avi", video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, skip*2, sample_length)
cmd3 = build_compression_command (video_source_filename, "compression_test_3.avi", video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, skip*4, sample_length)
cmd4 = build_compression_command (video_source_filename, "compression_test_4.avi", video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, skip*6, sample_length)
cmd5 = build_compression_command (video_source_filename, "compression_test_5.avi", video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, skip*8, sample_length)
run(cmd1)
run(cmd2)
run(cmd3)
run(cmd4)
run(cmd5)
size = get_filesize ("compression_test_1.avi")+get_filesize ("compression_test_2.avi")+get_filesize ("compression_test_3.avi")+get_filesize ("compression_test_4.avi")+get_filesize ("compression_test_5.avi")
return (size / 5.0)
def compress_video (video_source_filename, video_final_filename, video_target_size, audio_id=128, video_bitrate=1000, video_codec='mpeg4', audio_codec='mp3', video_fourcc_override='FMP4', video_gray_flag=0, video_crop_area=None, video_aspect_ratio='16/9', video_scale=None, video_encode_passes=2, video_deinterlace_flag=0, audio_volume_boost=None, audio_sample_rate=None, audio_bitrate=None, seek_skip=None, seek_length=None, video_chapter=None, verbose_flag=0, dry_run_flag=0):
"""This compresses the video and audio of the given source video filename to the transcoded filename.
This does a two-pass compression (I'm assuming mpeg4, I should probably make this smarter for other formats).
"""
#
# do the first pass video compression
#
#cmd = "mencoder -quiet '%(video_source_filename)s' -ss 65 -endpos 20 -aid %(audio_id)s -o '%(video_final_filename)s' -ffourcc %(video_fourcc_override)s -ovc lavc -oac lavc %(lavcopts)s %(video_filter)s %(audio_filter)s" % locals()
cmd = build_compression_command (video_source_filename, video_final_filename, video_target_size, audio_id, video_bitrate, video_codec, audio_codec, video_fourcc_override, video_gray_flag, video_crop_area, video_aspect_ratio, video_scale, video_encode_passes, video_deinterlace_flag, audio_volume_boost, audio_sample_rate, audio_bitrate, seek_skip, seek_length, video_chapter)
if verbose_flag: print cmd
if not dry_run_flag:
run(cmd)
print
# If not doing two passes then return early.
if video_encode_passes!='2':
return
if verbose_flag:
video_actual_size = get_filesize (video_final_filename)
if video_actual_size > video_target_size:
print "======================================================="
print "WARNING!"
print "First pass compression resulted in"
print "actual file size greater than target size."
print "Second pass will be too big."
print "======================================================="
#
# do the second pass video compression
#
cmd = cmd.replace ('vpass=1', 'vpass=2')
if verbose_flag: print cmd
if not dry_run_flag:
run(cmd)
print
return
def compress_audio (audio_raw_filename, audio_compressed_filename, audio_lowpass_filter=None, audio_sample_rate=None, audio_bitrate=None, verbose_flag=0, dry_run_flag=0):
"""This is depricated.
This compresses the raw audio file to the compressed audio filename.
"""
cmd = 'lame -h --athaa-sensitivity 1' # --cwlimit 11"
if audio_lowpass_filter:
cmd = cmd + ' --lowpass ' + audio_lowpass_filter
if audio_bitrate:
#cmd = cmd + ' --abr ' + audio_bitrate
cmd = cmd + ' --cbr -b ' + audio_bitrate
if audio_sample_rate:
cmd = cmd + ' --resample ' + audio_sample_rate
cmd = cmd + ' ' + audio_raw_filename + ' ' + audio_compressed_filename
if verbose_flag: print cmd
if not dry_run_flag:
(command_output, exitstatus) = run(cmd)
print
if exitstatus != 0:
raise Exception('ERROR: lame failed to compress raw audio file.')
def mux (video_final_filename, video_transcoded_filename, audio_compressed_filename, video_container_format, verbose_flag=0, dry_run_flag=0):
"""This is depricated. I used to use a three-pass encoding where I would mix the audio track separately, but
this never worked very well (loss of audio sync)."""
if video_container_format.lower() == 'mkv': # Matroska
mux_mkv (video_final_filename, video_transcoded_filename, audio_compressed_filename, verbose_flag, dry_run_flag)
if video_container_format.lower() == 'avi':
mux_avi (video_final_filename, video_transcoded_filename, audio_compressed_filename, verbose_flag, dry_run_flag)
def mux_mkv (video_final_filename, video_transcoded_filename, audio_compressed_filename, verbose_flag=0, dry_run_flag=0):
"""This is depricated."""
cmd = 'mkvmerge -o %s --noaudio %s %s' % (video_final_filename, video_transcoded_filename, audio_compressed_filename)
if verbose_flag: print cmd
if not dry_run_flag:
run(cmd)
print
def mux_avi (video_final_filename, video_transcoded_filename, audio_compressed_filename, verbose_flag=0, dry_run_flag=0):
"""This is depricated."""
pass
# cmd = "mencoder -quiet -oac copy -ovc copy -o '%s' -audiofile %s '%s'" % (video_final_filename, audio_compressed_filename, video_transcoded_filename)
# if verbose_flag: print cmd
# if not dry_run_flag:
# run(cmd)
# print
def delete_tmp_files (audio_raw_filename, verbose_flag=0, dry_run_flag=0):
global GLOBAL_LOGFILE_NAME
file_list = ' '.join([GLOBAL_LOGFILE_NAME, 'divx2pass.log', audio_raw_filename ])
cmd = 'rm -f ' + file_list
if verbose_flag: print cmd
if not dry_run_flag:
run(cmd)
print
##############################################################################
# This is the interactive Q&A that is used if a conf file was not given.
##############################################################################
def interactive_convert ():
global prompts, prompts_key_order
print globals()['__doc__']
print
print "=============================================="
print " Enter '?' at any question to get extra help."
print "=============================================="
print
# Ask for the level of options the user wants.
# A lot of code just to print a string!
level_sort = {0:'', 1:'', 2:''}
for k in prompts:
level = prompts[k][3]
if level < 0 or level > 2:
continue
level_sort[level] += " " + prompts[k][1] + "\n"
level_sort_string = "This sets the level for advanced options prompts. Set 0 for simple, 1 for advanced, or 2 for expert.\n"
level_sort_string += "[0] Basic options:\n" + str(level_sort[0]) + "\n"
level_sort_string += "[1] Advanced options:\n" + str(level_sort[1]) + "\n"
level_sort_string += "[2] Expert options:\n" + str(level_sort[2])
c = input_option("Prompt level (0, 1, or 2)?", "1", level_sort_string)
max_prompt_level = int(c)
options = {}
for k in prompts_key_order:
if k == 'video_aspect_ratio':
guess_aspect = get_aspect_ratio(options['video_source_filename'])
options[k] = input_option (prompts[k][1], guess_aspect, prompts[k][2], prompts[k][3], max_prompt_level)
elif k == 'audio_id':
aid_list = get_aid_list (options['video_source_filename'])
default_id = '128'
if max_prompt_level>=prompts[k][3]:
if len(aid_list) > 1:
print "This video has more than one audio stream. The following stream audio IDs were found:"
for aid in aid_list:
print " " + aid
default_id = aid_list[0]
else:
print "WARNING!"
print "Rippy was unable to get the list of audio streams from this video."
print "If reading directly from a DVD then the DVD device might be busy."
print "Using a default setting of stream id 128 (main audio on most DVDs)."
default_id = '128'
options[k] = input_option (prompts[k][1], default_id, prompts[k][2], prompts[k][3], max_prompt_level)
elif k == 'subtitle_id':
sid_list = get_sid_list (options['video_source_filename'])
default_id = 'None'
if max_prompt_level>=prompts[k][3]:
if len(sid_list) > 0:
print "This video has one or more subtitle streams. The following stream subtitle IDs were found:"
for sid in sid_list:
print " " + sid
#default_id = sid_list[0]
default_id = prompts[k][0]
else:
print "WARNING!"
print "Unable to get the list of subtitle streams from this video. It may have none."
print "Setting default to None."
default_id = 'None'
options[k] = input_option (prompts[k][1], default_id, prompts[k][2], prompts[k][3], max_prompt_level)
elif k == 'audio_lowpass_filter':
lowpass_default = "%.1f" % (math.floor(float(options['audio_sample_rate']) / 2.0))
options[k] = input_option (prompts[k][1], lowpass_default, prompts[k][2], prompts[k][3], max_prompt_level)
elif k == 'video_bitrate':
if options['video_length'].lower() == 'none':
options[k] = input_option (prompts[k][1], '1000', prompts[k][2], prompts[k][3], max_prompt_level)
else:
options[k] = input_option (prompts[k][1], prompts[k][0], prompts[k][2], prompts[k][3], max_prompt_level)
else:
# don't bother asking for video_target_size or video_bitrate_overhead if video_bitrate was set
if (k=='video_target_size' or k=='video_bitrate_overhead') and options['video_bitrate']!='calc':
continue
# don't bother with crop area if video length is none
if k == 'video_crop_area' and options['video_length'].lower() == 'none':
options['video_crop_area'] = 'none'
continue
options[k] = input_option (prompts[k][1], prompts[k][0], prompts[k][2], prompts[k][3], max_prompt_level)
#options['video_final_filename'] = options['video_final_filename'] + "." + options['video_container_format']
print "=========================================================================="
print "Ready to Rippy!"
print
print "The following options will be used:"
for k,v in options.iteritems():
print "%27s : %s" % (k, v)
print
c = input_option("Continue?", "Y")
c = c.strip().lower()
if c[0] != 'y':
print "Exiting..."
os._exit(1)
return options
def clean_options (d):
"""This validates and cleans up the options dictionary.
After reading options interactively or from a conf file
we need to make sure that the values make sense and are
converted to the correct type.
1. Any key with "_flag" in it becomes a boolean True or False.
2. Values are normalized ("No", "None", "none" all become "none";
"Calcluate", "c", "CALC" all become "calc").
3. Certain values are converted from string to int.
4. Certain combinations of options are invalid or override each other.
This is a rather annoying function, but then so it most cleanup work.
"""
for k in d:
d[k] = d[k].strip()
# convert all flag options to 0 or 1
if '_flag' in k:
if type(d[k]) is types.StringType:
if d[k].strip().lower()[0] in 'yt1': #Yes, True, 1
d[k] = 1
else:
d[k] = 0
d['video_bitrate'] = d['video_bitrate'].lower()
if d['video_bitrate'][0]=='c':
d['video_bitrate']='calc'
else:
d['video_bitrate'] = int(float(d['video_bitrate']))
try:
d['video_target_size'] = int(d['video_target_size'])
# shorthand magic numbers get automatically expanded
if d['video_target_size'] == 180:
d['video_target_size'] = 193536000
elif d['video_target_size'] == 550:
d['video_target_size'] = 580608000
elif d['video_target_size'] == 650:
d['video_target_size'] = 681984000
elif d['video_target_size'] == 700:
d['video_target_size'] = 737280000
except:
d['video_target_size'] = 'none'
try:
d['video_chapter'] = int(d['video_chapter'])
except:
d['video_chapter'] = None
try:
d['subtitle_id'] = int(d['subtitle_id'])
except:
d['subtitle_id'] = None
try:
d['video_bitrate_overhead'] = float(d['video_bitrate_overhead'])
except:
d['video_bitrate_overhead'] = -1.0
d['audio_bitrate'] = int(d['audio_bitrate'])
d['audio_sample_rate'] = int(d['audio_sample_rate'])
d['audio_volume_boost'] = d['audio_volume_boost'].lower()
if d['audio_volume_boost'][0]=='n':
d['audio_volume_boost'] = None
else:
d['audio_volume_boost'] = d['audio_volume_boost'].replace('db','')
d['audio_volume_boost'] = float(d['audio_volume_boost'])
# assert (d['video_bitrate']=='calc' and d['video_target_size']!='none')
# or (d['video_bitrate']!='calc' and d['video_target_size']=='none')
d['video_scale'] = d['video_scale'].lower()
if d['video_scale'][0]=='n':
d['video_scale']='none'
else:
al = re.findall("([0-9]+).*?([0-9]+)", d['video_scale'])
d['video_scale']=al[0][0]+':'+al[0][1]
d['video_crop_area'] = d['video_crop_area'].lower()
if d['video_crop_area'][0]=='n':
d['video_crop_area']='none'
d['video_length'] = d['video_length'].lower()
if d['video_length'][0]=='c':
d['video_length']='calc'
elif d['video_length'][0]=='n':
d['video_length']='none'
else:
d['video_length'] = int(float(d['video_length']))
if d['video_length']==0:
d['video_length'] = 'none'
assert (not (d['video_length']=='none' and d['video_bitrate']=='calc'))
return d
def main ():
try:
optlist, args = getopt.getopt(sys.argv[1:], 'h?', ['help','h','?'])
except Exception, e:
print str(e)
exit_with_usage()
command_line_options = dict(optlist)
# There are a million ways to cry for help. These are but a few of them.
if [elem for elem in command_line_options if elem in ['-h','--h','-?','--?','--help']]:
exit_with_usage(0)
missing = check_missing_requirements()
if missing is not None:
print
print "=========================================================================="
print "ERROR!"
print "Some required external commands are missing."
print "please install the following packages:"
print str(missing)
print "=========================================================================="
print
c = input_option("Continue?", "Y")
c = c.strip().lower()
if c[0] != 'y':
print "Exiting..."
os._exit(1)
if len(args) > 0:
# cute one-line string-to-dictionary parser (two-lines if you count this comment):
options = dict(re.findall('([^: \t\n]*)\s*:\s*(".*"|[^ \t\n]*)', file(args[0]).read()))
options = clean_options(options)
convert (options)
else:
options = interactive_convert ()
options = clean_options(options)
convert (options)
print "# Done!"
if __name__ == "__main__":
try:
start_time = time.time()
print time.asctime()
main()
print time.asctime()
print "TOTAL TIME IN MINUTES:",
print (time.time() - start_time) / 60.0
except Exception, e:
tb_dump = traceback.format_exc()
print "=========================================================================="
print "ERROR -- Unexpected exception in script."
print str(e)
print str(tb_dump)
print "=========================================================================="
print >>GLOBAL_LOGFILE, "=========================================================================="
print >>GLOBAL_LOGFILE, "ERROR -- Unexpected exception in script."
print >>GLOBAL_LOGFILE, str(e)
print >>GLOBAL_LOGFILE, str(tb_dump)
print >>GLOBAL_LOGFILE, "=========================================================================="
exit_with_usage(3)

View File

@ -0,0 +1,103 @@
#!/usr/bin/env python
"""This spawns a sub-shell (bash) and gives the user interactive control. The
entire shell session is logged to a file called script.log. This behaves much
like the classic BSD command 'script'.
./script.py [-a] [-c command] {logfilename}
logfilename : This is the name of the log file. Default is script.log.
-a : Append to log file. Default is to overwrite log file.
-c : spawn command. Default is to spawn the sh shell.
Example:
This will start a bash shell and append to the log named my_session.log:
./script.py -a -c bash my_session.log
"""
import os, sys, time, getopt
import signal, fcntl, termios, struct
import traceback
import pexpect
global_pexpect_instance = None # Used by signal handler
def exit_with_usage():
print globals()['__doc__']
os._exit(1)
def main():
######################################################################
# Parse the options, arguments, get ready, etc.
######################################################################
try:
optlist, args = getopt.getopt(sys.argv[1:], 'h?ac:', ['help','h','?'])
except Exception, e:
print str(e)
exit_with_usage()
options = dict(optlist)
if len(args) > 1:
exit_with_usage()
if [elem for elem in options if elem in ['-h','--h','-?','--?','--help']]:
print "Help:"
exit_with_usage()
if len(args) == 1:
script_filename = args[0]
else:
script_filename = "script.log"
if '-a' in options:
fout = file (script_filename, "ab")
else:
fout = file (script_filename, "wb")
if '-c' in options:
command = options['-c']
else:
command = "sh"
# Begin log with date/time in the form CCCCyymm.hhmmss
fout.write ('# %4d%02d%02d.%02d%02d%02d \n' % time.localtime()[:-3])
######################################################################
# Start the interactive session
######################################################################
p = pexpect.spawn(command)
p.logfile = fout
global global_pexpect_instance
global_pexpect_instance = p
signal.signal(signal.SIGWINCH, sigwinch_passthrough)
print "Script recording started. Type ^] (ASCII 29) to escape from the script shell."
p.interact(chr(29))
fout.close()
return 0
def sigwinch_passthrough (sig, data):
# Check for buggy platforms (see pexpect.setwinsize()).
if 'TIOCGWINSZ' in dir(termios):
TIOCGWINSZ = termios.TIOCGWINSZ
else:
TIOCGWINSZ = 1074295912 # assume
s = struct.pack ("HHHH", 0, 0, 0, 0)
a = struct.unpack ('HHHH', fcntl.ioctl(sys.stdout.fileno(), TIOCGWINSZ , s))
global global_pexpect_instance
global_pexpect_instance.setwinsize(a[0],a[1])
if __name__ == "__main__":
try:
main()
except SystemExit, e:
raise e
except Exception, e:
print "ERROR"
print str(e)
traceback.print_exc()
os._exit(1)

View File

@ -0,0 +1,94 @@
#
# Eric S. Raymond
#
# Greatly modified by Nigel W. Moriarty
# April 2003
#
from pexpect import *
import os, sys
import getpass
import time
class ssh_session:
"Session with extra state including the password to be used."
def __init__(self, user, host, password=None, verbose=0):
self.user = user
self.host = host
self.verbose = verbose
self.password = password
self.keys = [
'authenticity',
'assword:',
'@@@@@@@@@@@@',
'Command not found.',
EOF,
]
self.f = open('ssh.out','w')
def __repr__(self):
outl = 'class :'+self.__class__.__name__
for attr in self.__dict__:
if attr == 'password':
outl += '\n\t'+attr+' : '+'*'*len(self.password)
else:
outl += '\n\t'+attr+' : '+str(getattr(self, attr))
return outl
def __exec(self, command):
"Execute a command on the remote host. Return the output."
child = spawn(command,
#timeout=10,
)
if self.verbose:
sys.stderr.write("-> " + command + "\n")
seen = child.expect(self.keys)
self.f.write(str(child.before) + str(child.after)+'\n')
if seen == 0:
child.sendline('yes')
seen = child.expect(self.keys)
if seen == 1:
if not self.password:
self.password = getpass.getpass('Remote password: ')
child.sendline(self.password)
child.readline()
time.sleep(5)
# Added to allow the background running of remote process
if not child.isalive():
seen = child.expect(self.keys)
if seen == 2:
lines = child.readlines()
self.f.write(lines)
if self.verbose:
sys.stderr.write("<- " + child.before + "|\n")
try:
self.f.write(str(child.before) + str(child.after)+'\n')
except:
pass
self.f.close()
return child.before
def ssh(self, command):
return self.__exec("ssh -l %s %s \"%s\"" \
% (self.user,self.host,command))
def scp(self, src, dst):
return self.__exec("scp %s %s@%s:%s" \
% (src, session.user, session.host, dst))
def exists(self, file):
"Retrieve file permissions of specified remote file."
seen = self.ssh("/bin/ls -ld %s" % file)
if string.find(seen, "No such file") > -1:
return None # File doesn't exist
else:
return seen.split()[0] # Return permission field of listing.

View File

@ -0,0 +1,72 @@
#!/usr/bin/env python
"""This starts an SSH tunnel to a given host. If the SSH process ever dies then
this script will detect that and restart it. I use this under Cygwin to keep
open encrypted tunnels to port 25 (SMTP), port 143 (IMAP4), and port 110
(POP3). I set my mail client to talk to localhost and I keep this script
running in the background.
Note that this is a rather stupid script at the moment because it just looks to
see if any ssh process is running. It should really make sure that our specific
ssh process is running. The problem is that ssh is missing a very useful
feature. It has no way to report the process id of the background daemon that
it creates with the -f command. This would be a really useful script if I could
figure a way around this problem. """
import pexpect
import getpass
import time
# SMTP:25 IMAP4:143 POP3:110
tunnel_command = 'ssh -C -N -f -L 25:127.0.0.1:25 -L 143:127.0.0.1:143 -L 110:127.0.0.1:110 %(user)@%(host)'
host = raw_input('Hostname: ')
user = raw_input('Username: ')
X = getpass.getpass('Password: ')
def get_process_info ():
# This seems to work on both Linux and BSD, but should otherwise be considered highly UNportable.
ps = pexpect.run ('ps ax -O ppid')
pass
def start_tunnel ():
try:
ssh_tunnel = pexpect.spawn (tunnel_command % globals())
ssh_tunnel.expect ('password:')
time.sleep (0.1)
ssh_tunnel.sendline (X)
time.sleep (60) # Cygwin is slow to update process status.
ssh_tunnel.expect (pexpect.EOF)
except Exception, e:
print str(e)
def main ():
while True:
ps = pexpect.spawn ('ps')
time.sleep (1)
index = ps.expect (['/usr/bin/ssh', pexpect.EOF, pexpect.TIMEOUT])
if index == 2:
print 'TIMEOUT in ps command...'
print str(ps)
time.sleep (13)
if index == 1:
print time.asctime(),
print 'restarting tunnel'
start_tunnel ()
time.sleep (11)
print 'tunnel OK'
else:
# print 'tunnel OK'
time.sleep (7)
if __name__ == '__main__':
main ()
# This was for older SSH versions that didn't have -f option
#tunnel_command = 'ssh -C -n -L 25:%(host)s:25 -L 110:%(host)s:110 %(user)s@%(host)s -f nothing.sh'
#nothing_script = """#!/bin/sh
#while true; do sleep 53; done
#"""

View File

@ -0,0 +1,56 @@
#!/usr/bin/env python
"""This runs 'ls -l' on a remote host using SSH. At the prompts enter hostname,
user, and password.
$Id: sshls.py 489 2007-11-28 23:40:34Z noah $
"""
import pexpect
import getpass, os
def ssh_command (user, host, password, command):
"""This runs a command on the remote host. This could also be done with the
pxssh class, but this demonstrates what that class does at a simpler level.
This returns a pexpect.spawn object. This handles the case when you try to
connect to a new host and ssh asks you if you want to accept the public key
fingerprint and continue connecting. """
ssh_newkey = 'Are you sure you want to continue connecting'
child = pexpect.spawn('ssh -l %s %s %s'%(user, host, command))
i = child.expect([pexpect.TIMEOUT, ssh_newkey, 'password: '])
if i == 0: # Timeout
print 'ERROR!'
print 'SSH could not login. Here is what SSH said:'
print child.before, child.after
return None
if i == 1: # SSH does not have the public key. Just accept it.
child.sendline ('yes')
child.expect ('password: ')
i = child.expect([pexpect.TIMEOUT, 'password: '])
if i == 0: # Timeout
print 'ERROR!'
print 'SSH could not login. Here is what SSH said:'
print child.before, child.after
return None
child.sendline(password)
return child
def main ():
host = raw_input('Hostname: ')
user = raw_input('User: ')
password = getpass.getpass('Password: ')
child = ssh_command (user, host, password, '/bin/ls -l')
child.expect(pexpect.EOF)
print child.before
if __name__ == '__main__':
try:
main()
except Exception, e:
print str(e)
traceback.print_exc()
os._exit(1)

View File

@ -0,0 +1,106 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<html>
<head>
<title>TEST</title>
</head>
<style type="text/css">
a {color: #9f9; text-decoration: none}
a:hover {color: #0f0}
hr {color: #0f0}
html,table,body,textarea,input,form
{
font-family: "Courier New", Courier, mono;
font-size: 8pt;
color: #0c0;
background-color: #020;
margin:0;
padding:0;
border:0;
}
input { background-color: #010; }
textarea {
border-width:1;
border-style:solid;
border-color:#0c0;
padding:3;
margin:3;
}
</style>
<script>
var foo="" +
" 123456789012345678901234567890123456789012345 789012345678901234567890123456789"+
"0 2345678901234567890123456789012345678901234 6 89012345678901234567890123456789"+
"01 34567890123456789012345678901234567890123 567 9012345678901234567890123456789"+
"012 456789012345678901234567890123456789012 45678 012345678901234567890123456789"+
"0123 5678901234567890123456789012345678901 3456789 12345678901234567890123456789"+
"01234 67890123456789012345678901234567890 234567890 2345678901234567890123456789"+
"012345 789012345678901234567890123456789 12345678901 345678901234567890123456789"+
"0123456 8901234567890123456789012345678 0123456789012 45678901234567890123456789"+
"01234567 90123456789012345678901234567 901234567890123 5678901234567890123456789"+
"012345678 012345678901234567890123456 89012345678901234 678901234567890123456789"+
"0123456789 1234567890123456789012345 7890123456789012345 78901234567890123456789"+
"01234567890 23456789012345678901234 678901234567890123456 8901234567890123456789"+
"012345678901 345678901234567890123 56789012345678901234567 901234567890123456789"+
"0123456789012 4567890123456789012 4567890123456789012345678 0123456789012345678 "+
"01234567890123 56789012345678901 345678901234567890123456789 12345678901234567 9"+
"012345678901234 678901234567890 23456789012 567 01234567890 234567890123456 89"+
"0123456789012345 7890123456789 123457789012 567 012345678901 3456789012345 789"+
"01234567890123456 89012345678 012345678901234567890123456789012 45678901234 6789"+
"012345678901234567 901234567 90123456789 12345678901 34567890123 567890123 56789"+
"0123456789012345678 0123456 8901234567890 3456789 2345678901234 6789012 456789"+
"01234567890123456789 12345 7890123456789012 0123456789012345 78901 3456789"+
"012345678901234567890 234 67890123456789012345678901234567890123456 890 23456789"+
"0123456789012345678901 3 5678901234567890123456789012345678901234567 9 123456789"+
"01234567890123456789012 456789012345678901234567890123456789012345678 0123456789";
function start2()
{
// get the reference for the body
//var mybody = document.getElementsByTagName("body")[0];
var mybody = document.getElementById("replace_me");
var myroot = document.getElementById("a_parent");
mytable = document.createElement("table");
mytablebody = document.createElement("tbody");
mytable.setAttribute("border","0");
mytable.setAttribute("cellspacing","0");
mytable.setAttribute("cellpadding","0");
for(var j = 0; j < 24; j++)
{
mycurrent_row = document.createElement("tr");
for(var i = 0; i < 80; i++)
{
mycurrent_cell = document.createElement("td");
offset = (j*80)+i;
currenttext = document.createTextNode(foo.substring(offset,offset+1));
mycurrent_cell.appendChild(currenttext);
mycurrent_row.appendChild(mycurrent_cell);
}
mytablebody.appendChild(mycurrent_row);
}
mytable.appendChild(mytablebody);
myroot.replaceChild(mytable,mybody);
//mybody.appendChild(mytable);
}
</script>
<body onload="start2();">
<table align="LEFT" border="0" cellspacing="0" cellpadding="0">
<div id="a_parent">
<span id="replace_me">
<tr align="left" valign="left">
<td>/</td>
<td>h</td>
<td>o</td>
<td>m</td>
<td>e</td>
<td>/</td>
<td>n</td>
<td>o</td>
<td>a</td>
<td>h</td>
<td>/</td>
<td>&nbsp;</td>
</tr>
</table>
</span>
</div>
</body>
</html>

View File

@ -0,0 +1,267 @@
#!/usr/bin/env python
""" This runs netstat on a local or remote server. It calculates some simple
statistical information on the number of external inet connections. It groups
by IP address. This can be used to detect if one IP address is taking up an
excessive number of connections. It can also send an email alert if a given IP
address exceeds a threshold between runs of the script. This script can be used
as a drop-in Munin plugin or it can be used stand-alone from cron. I used this
on a busy web server that would sometimes get hit with denial of service
attacks. This made it easy to see if a script was opening many multiple
connections. A typical browser would open fewer than 10 connections at once. A
script might open over 100 simultaneous connections.
./topip.py [-s server_hostname] [-u username] [-p password] {-a from_addr,to_addr} {-n N} {-v} {--ipv6}
-s : hostname of the remote server to login to.
-u : username to user for login.
-p : password to user for login.
-n : print stddev for the the number of the top 'N' ipaddresses.
-v : verbose - print stats and list of top ipaddresses.
-a : send alert if stddev goes over 20.
-l : to log message to /var/log/topip.log
--ipv6 : this parses netstat output that includes ipv6 format.
Note that this actually only works with ipv4 addresses, but for versions of
netstat that print in ipv6 format.
--stdev=N : Where N is an integer. This sets the trigger point for alerts and logs.
Default is to trigger if max value is above 5 standard deviations.
Example:
This will print stats for the top IP addresses connected to the given host:
./topip.py -s www.example.com -u mylogin -p mypassword -n 10 -v
This will send an alert email if the maxip goes over the stddev trigger value and
the the current top ip is the same as the last top ip (/tmp/topip.last):
./topip.py -s www.example.com -u mylogin -p mypassword -n 10 -v -a alert@example.com,user@example.com
This will print the connection stats for the localhost in Munin format:
./topip.py
Noah Spurrier
$Id: topip.py 489 2007-11-28 23:40:34Z noah $
"""
import pexpect, pxssh # See http://pexpect.sourceforge.net/
import os, sys, time, re, getopt, pickle, getpass, smtplib
import traceback
from pprint import pprint
TOPIP_LOG_FILE = '/var/log/topip.log'
TOPIP_LAST_RUN_STATS = '/var/run/topip.last'
def exit_with_usage():
print globals()['__doc__']
os._exit(1)
def stats(r):
"""This returns a dict of the median, average, standard deviation, min and max of the given sequence.
>>> from topip import stats
>>> print stats([5,6,8,9])
{'med': 8, 'max': 9, 'avg': 7.0, 'stddev': 1.5811388300841898, 'min': 5}
>>> print stats([1000,1006,1008,1014])
{'med': 1008, 'max': 1014, 'avg': 1007.0, 'stddev': 5.0, 'min': 1000}
>>> print stats([1,3,4,5,18,16,4,3,3,5,13])
{'med': 4, 'max': 18, 'avg': 6.8181818181818183, 'stddev': 5.6216817577237475, 'min': 1}
>>> print stats([1,3,4,5,18,16,4,3,3,5,13,14,5,6,7,8,7,6,6,7,5,6,4,14,7])
{'med': 6, 'max': 18, 'avg': 7.0800000000000001, 'stddev': 4.3259218670706474, 'min': 1}
"""
total = sum(r)
avg = float(total)/float(len(r))
sdsq = sum([(i-avg)**2 for i in r])
s = list(r)
s.sort()
return dict(zip(['med', 'avg', 'stddev', 'min', 'max'] , (s[len(s)//2], avg, (sdsq/len(r))**.5, min(r), max(r))))
def send_alert (message, subject, addr_from, addr_to, smtp_server='localhost'):
"""This sends an email alert.
"""
message = 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n' % (addr_from, addr_to, subject) + message
server = smtplib.SMTP(smtp_server)
server.sendmail(addr_from, addr_to, message)
server.quit()
def main():
######################################################################
## Parse the options, arguments, etc.
######################################################################
try:
optlist, args = getopt.getopt(sys.argv[1:], 'h?valqs:u:p:n:', ['help','h','?','ipv6','stddev='])
except Exception, e:
print str(e)
exit_with_usage()
options = dict(optlist)
munin_flag = False
if len(args) > 0:
if args[0] == 'config':
print 'graph_title Netstat Connections per IP'
print 'graph_vlabel Socket connections per IP'
print 'connections_max.label max'
print 'connections_max.info Maximum number of connections per IP'
print 'connections_avg.label avg'
print 'connections_avg.info Average number of connections per IP'
print 'connections_stddev.label stddev'
print 'connections_stddev.info Standard deviation'
return 0
elif args[0] != '':
print args, len(args)
return 0
exit_with_usage()
if [elem for elem in options if elem in ['-h','--h','-?','--?','--help']]:
print 'Help:'
exit_with_usage()
if '-s' in options:
hostname = options['-s']
else:
# if host was not specified then assume localhost munin plugin.
munin_flag = True
hostname = 'localhost'
# If localhost then don't ask for username/password.
if hostname != 'localhost' and hostname != '127.0.0.1':
if '-u' in options:
username = options['-u']
else:
username = raw_input('username: ')
if '-p' in options:
password = options['-p']
else:
password = getpass.getpass('password: ')
else:
use_localhost = True
if '-l' in options:
log_flag = True
else:
log_flag = False
if '-n' in options:
average_n = int(options['-n'])
else:
average_n = None
if '-v' in options:
verbose = True
else:
verbose = False
if '-a' in options:
alert_flag = True
(alert_addr_from, alert_addr_to) = tuple(options['-a'].split(','))
else:
alert_flag = False
if '--ipv6' in options:
ipv6_flag = True
else:
ipv6_flag = False
if '--stddev' in options:
stddev_trigger = float(options['--stddev'])
else:
stddev_trigger = 5
if ipv6_flag:
netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+::ffff:(\S+):(\S+)\s+.*?\r'
else:
netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(?:::ffff:)*(\S+):(\S+)\s+.*?\r'
#netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+):(\S+)\s+.*?\r'
# run netstat (either locally or via SSH).
if use_localhost:
p = pexpect.spawn('netstat -n -t')
PROMPT = pexpect.TIMEOUT
else:
p = pxssh.pxssh()
p.login(hostname, username, password)
p.sendline('netstat -n -t')
PROMPT = p.PROMPT
# loop through each matching netstat_pattern and put the ip address in the list.
ip_list = {}
try:
while 1:
i = p.expect([PROMPT, netstat_pattern])
if i == 0:
break
k = p.match.groups()[4]
if k in ip_list:
ip_list[k] = ip_list[k] + 1
else:
ip_list[k] = 1
except:
pass
# remove a few common, uninteresting addresses from the dictionary.
ip_list = dict([ (key,value) for key,value in ip_list.items() if '192.168.' not in key])
ip_list = dict([ (key,value) for key,value in ip_list.items() if '127.0.0.1' not in key])
# sort dict by value (count)
#ip_list = sorted(ip_list.iteritems(),lambda x,y:cmp(x[1], y[1]),reverse=True)
ip_list = ip_list.items()
if len(ip_list) < 1:
if verbose: print 'Warning: no networks connections worth looking at.'
return 0
ip_list.sort(lambda x,y:cmp(y[1],x[1]))
# generate some stats for the ip addresses found.
if average_n <= 1:
average_n = None
s = stats(zip(*ip_list[0:average_n])[1]) # The * unary operator treats the list elements as arguments
s['maxip'] = ip_list[0]
# print munin-style or verbose results for the stats.
if munin_flag:
print 'connections_max.value', s['max']
print 'connections_avg.value', s['avg']
print 'connections_stddev.value', s['stddev']
return 0
if verbose:
pprint (s)
print
pprint (ip_list[0:average_n])
# load the stats from the last run.
try:
last_stats = pickle.load(file(TOPIP_LAST_RUN_STATS))
except:
last_stats = {'maxip':None}
if s['maxip'][1] > (s['stddev'] * stddev_trigger) and s['maxip']==last_stats['maxip']:
if verbose: print 'The maxip has been above trigger for two consecutive samples.'
if alert_flag:
if verbose: print 'SENDING ALERT EMAIL'
send_alert(str(s), 'ALERT on %s' % hostname, alert_addr_from, alert_addr_to)
if log_flag:
if verbose: print 'LOGGING THIS EVENT'
fout = file(TOPIP_LOG_FILE,'a')
#dts = time.strftime('%Y:%m:%d:%H:%M:%S', time.localtime())
dts = time.asctime()
fout.write ('%s - %d connections from %s\n' % (dts,s['maxip'][1],str(s['maxip'][0])))
fout.close()
# save state to TOPIP_LAST_RUN_STATS
try:
pickle.dump(s, file(TOPIP_LAST_RUN_STATS,'w'))
os.chmod (TOPIP_LAST_RUN_STATS, 0664)
except:
pass
# p.logout()
if __name__ == '__main__':
try:
main()
sys.exit(0)
except SystemExit, e:
raise e
except Exception, e:
print str(e)
traceback.print_exc()
os._exit(1)

View File

@ -0,0 +1,57 @@
#!/usr/bin/env python
"""This displays uptime information using uptime. This is redundant,
but it demonstrates expecting for a regular expression that uses subgroups.
$Id: uptime.py 489 2007-11-28 23:40:34Z noah $
"""
import pexpect
import re
# There are many different styles of uptime results. I try to parse them all. Yeee!
# Examples from different machines:
# [x86] Linux 2.4 (Redhat 7.3)
# 2:06pm up 63 days, 18 min, 3 users, load average: 0.32, 0.08, 0.02
# [x86] Linux 2.4.18-14 (Redhat 8.0)
# 3:07pm up 29 min, 1 user, load average: 2.44, 2.51, 1.57
# [PPC - G4] MacOS X 10.1 SERVER Edition
# 2:11PM up 3 days, 13:50, 3 users, load averages: 0.01, 0.00, 0.00
# [powerpc] Darwin v1-58.corefa.com 8.2.0 Darwin Kernel Version 8.2.0
# 10:35 up 18:06, 4 users, load averages: 0.52 0.47 0.36
# [Sparc - R220] Sun Solaris (8)
# 2:13pm up 22 min(s), 1 user, load average: 0.02, 0.01, 0.01
# [x86] Linux 2.4.18-14 (Redhat 8)
# 11:36pm up 4 days, 17:58, 1 user, load average: 0.03, 0.01, 0.00
# AIX jwdir 2 5 0001DBFA4C00
# 09:43AM up 23:27, 1 user, load average: 0.49, 0.32, 0.23
# OpenBSD box3 2.9 GENERIC#653 i386
# 6:08PM up 4 days, 22:26, 1 user, load averages: 0.13, 0.09, 0.08
# This parses uptime output into the major groups using regex group matching.
p = pexpect.spawn ('uptime')
p.expect('up\s+(.*?),\s+([0-9]+) users?,\s+load averages?: ([0-9]+\.[0-9][0-9]),?\s+([0-9]+\.[0-9][0-9]),?\s+([0-9]+\.[0-9][0-9])')
duration, users, av1, av5, av15 = p.match.groups()
# The duration is a little harder to parse because of all the different
# styles of uptime. I'm sure there is a way to do this all at once with
# one single regex, but I bet it would be hard to read and maintain.
# If anyone wants to send me a version using a single regex I'd be happy to see it.
days = '0'
hours = '0'
mins = '0'
if 'day' in duration:
p.match = re.search('([0-9]+)\s+day',duration)
days = str(int(p.match.group(1)))
if ':' in duration:
p.match = re.search('([0-9]+):([0-9]+)',duration)
hours = str(int(p.match.group(1)))
mins = str(int(p.match.group(2)))
if 'min' in duration:
p.match = re.search('([0-9]+)\s+min',duration)
mins = str(int(p.match.group(1)))
# Print the parsed fields in CSV format.
print 'days, hours, minutes, users, cpu avg 1 min, cpu avg 5 min, cpu avg 15 min'
print '%s, %s, %s, %s, %s, %s, %s' % (days, hours, mins, users, av1, av5, av15)

View File

@ -0,0 +1,82 @@
"""This is like pexpect, but will work on any file descriptor that you pass it.
So you are reponsible for opening and close the file descriptor.
$Id: fdpexpect.py 505 2007-12-26 21:33:50Z noah $
"""
from pexpect import *
import os
__all__ = ['fdspawn']
class fdspawn (spawn):
"""This is like pexpect.spawn but allows you to supply your own open file
descriptor. For example, you could use it to read through a file looking
for patterns, or to control a modem or serial device. """
def __init__ (self, fd, args=[], timeout=30, maxread=2000, searchwindowsize=None, logfile=None):
"""This takes a file descriptor (an int) or an object that support the
fileno() method (returning an int). All Python file-like objects
support fileno(). """
### TODO: Add better handling of trying to use fdspawn in place of spawn
### TODO: (overload to allow fdspawn to also handle commands as spawn does.
if type(fd) != type(0) and hasattr(fd, 'fileno'):
fd = fd.fileno()
if type(fd) != type(0):
raise ExceptionPexpect ('The fd argument is not an int. If this is a command string then maybe you want to use pexpect.spawn.')
try: # make sure fd is a valid file descriptor
os.fstat(fd)
except OSError:
raise ExceptionPexpect, 'The fd argument is not a valid file descriptor.'
self.args = None
self.command = None
spawn.__init__(self, None, args, timeout, maxread, searchwindowsize, logfile)
self.child_fd = fd
self.own_fd = False
self.closed = False
self.name = '<file descriptor %d>' % fd
def __del__ (self):
return
def close (self):
if self.child_fd == -1:
return
if self.own_fd:
self.close (self)
else:
self.flush()
os.close(self.child_fd)
self.child_fd = -1
self.closed = True
def isalive (self):
"""This checks if the file descriptor is still valid. If os.fstat()
does not raise an exception then we assume it is alive. """
if self.child_fd == -1:
return False
try:
os.fstat(self.child_fd)
return True
except:
return False
def terminate (self, force=False):
raise ExceptionPexpect ('This method is not valid for file descriptors.')
def kill (self, sig):
return

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,311 @@
"""This class extends pexpect.spawn to specialize setting up SSH connections.
This adds methods for login, logout, and expecting the shell prompt.
$Id: pxssh.py 513 2008-02-09 18:26:13Z noah $
"""
from pexpect import *
import pexpect
import time
__all__ = ['ExceptionPxssh', 'pxssh']
# Exception classes used by this module.
class ExceptionPxssh(ExceptionPexpect):
"""Raised for pxssh exceptions.
"""
class pxssh (spawn):
"""This class extends pexpect.spawn to specialize setting up SSH
connections. This adds methods for login, logout, and expecting the shell
prompt. It does various tricky things to handle many situations in the SSH
login process. For example, if the session is your first login, then pxssh
automatically accepts the remote certificate; or if you have public key
authentication setup then pxssh won't wait for the password prompt.
pxssh uses the shell prompt to synchronize output from the remote host. In
order to make this more robust it sets the shell prompt to something more
unique than just $ or #. This should work on most Borne/Bash or Csh style
shells.
Example that runs a few commands on a remote server and prints the result::
import pxssh
import getpass
try:
s = pxssh.pxssh()
hostname = raw_input('hostname: ')
username = raw_input('username: ')
password = getpass.getpass('password: ')
s.login (hostname, username, password)
s.sendline ('uptime') # run a command
s.prompt() # match the prompt
print s.before # print everything before the prompt.
s.sendline ('ls -l')
s.prompt()
print s.before
s.sendline ('df')
s.prompt()
print s.before
s.logout()
except pxssh.ExceptionPxssh, e:
print "pxssh failed on login."
print str(e)
Note that if you have ssh-agent running while doing development with pxssh
then this can lead to a lot of confusion. Many X display managers (xdm,
gdm, kdm, etc.) will automatically start a GUI agent. You may see a GUI
dialog box popup asking for a password during development. You should turn
off any key agents during testing. The 'force_password' attribute will turn
off public key authentication. This will only work if the remote SSH server
is configured to allow password logins. Example of using 'force_password'
attribute::
s = pxssh.pxssh()
s.force_password = True
hostname = raw_input('hostname: ')
username = raw_input('username: ')
password = getpass.getpass('password: ')
s.login (hostname, username, password)
"""
def __init__ (self, timeout=30, maxread=2000, searchwindowsize=None, logfile=None, cwd=None, env=None):
spawn.__init__(self, None, timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, logfile=logfile, cwd=cwd, env=env)
self.name = '<pxssh>'
#SUBTLE HACK ALERT! Note that the command to set the prompt uses a
#slightly different string than the regular expression to match it. This
#is because when you set the prompt the command will echo back, but we
#don't want to match the echoed command. So if we make the set command
#slightly different than the regex we eliminate the problem. To make the
#set command different we add a backslash in front of $. The $ doesn't
#need to be escaped, but it doesn't hurt and serves to make the set
#prompt command different than the regex.
# used to match the command-line prompt
self.UNIQUE_PROMPT = "\[PEXPECT\][\$\#] "
self.PROMPT = self.UNIQUE_PROMPT
# used to set shell command-line prompt to UNIQUE_PROMPT.
self.PROMPT_SET_SH = "PS1='[PEXPECT]\$ '"
self.PROMPT_SET_CSH = "set prompt='[PEXPECT]\$ '"
self.SSH_OPTS = "-o'RSAAuthentication=no' -o 'PubkeyAuthentication=no'"
# Disabling X11 forwarding gets rid of the annoying SSH_ASKPASS from
# displaying a GUI password dialog. I have not figured out how to
# disable only SSH_ASKPASS without also disabling X11 forwarding.
# Unsetting SSH_ASKPASS on the remote side doesn't disable it! Annoying!
#self.SSH_OPTS = "-x -o'RSAAuthentication=no' -o 'PubkeyAuthentication=no'"
self.force_password = False
self.auto_prompt_reset = True
def levenshtein_distance(self, a,b):
"""This calculates the Levenshtein distance between a and b.
"""
n, m = len(a), len(b)
if n > m:
a,b = b,a
n,m = m,n
current = range(n+1)
for i in range(1,m+1):
previous, current = current, [i]+[0]*n
for j in range(1,n+1):
add, delete = previous[j]+1, current[j-1]+1
change = previous[j-1]
if a[j-1] != b[i-1]:
change = change + 1
current[j] = min(add, delete, change)
return current[n]
def sync_original_prompt (self):
"""This attempts to find the prompt. Basically, press enter and record
the response; press enter again and record the response; if the two
responses are similar then assume we are at the original prompt. This
is a slow function. It can take over 10 seconds. """
# All of these timing pace values are magic.
# I came up with these based on what seemed reliable for
# connecting to a heavily loaded machine I have.
# If latency is worse than these values then this will fail.
try:
self.read_nonblocking(size=10000,timeout=1) # GAS: Clear out the cache before getting the prompt
except TIMEOUT:
pass
time.sleep(0.1)
self.sendline()
time.sleep(0.5)
x = self.read_nonblocking(size=1000,timeout=1)
time.sleep(0.1)
self.sendline()
time.sleep(0.5)
a = self.read_nonblocking(size=1000,timeout=1)
time.sleep(0.1)
self.sendline()
time.sleep(0.5)
b = self.read_nonblocking(size=1000,timeout=1)
ld = self.levenshtein_distance(a,b)
len_a = len(a)
if len_a == 0:
return False
if float(ld)/len_a < 0.4:
return True
return False
### TODO: This is getting messy and I'm pretty sure this isn't perfect.
### TODO: I need to draw a flow chart for this.
def login (self,server,username,password='',terminal_type='ansi',original_prompt=r"[#$]",login_timeout=10,port=None,auto_prompt_reset=True):
"""This logs the user into the given server. It uses the
'original_prompt' to try to find the prompt right after login. When it
finds the prompt it immediately tries to reset the prompt to something
more easily matched. The default 'original_prompt' is very optimistic
and is easily fooled. It's more reliable to try to match the original
prompt as exactly as possible to prevent false matches by server
strings such as the "Message Of The Day". On many systems you can
disable the MOTD on the remote server by creating a zero-length file
called "~/.hushlogin" on the remote server. If a prompt cannot be found
then this will not necessarily cause the login to fail. In the case of
a timeout when looking for the prompt we assume that the original
prompt was so weird that we could not match it, so we use a few tricks
to guess when we have reached the prompt. Then we hope for the best and
blindly try to reset the prompt to something more unique. If that fails
then login() raises an ExceptionPxssh exception.
In some situations it is not possible or desirable to reset the
original prompt. In this case, set 'auto_prompt_reset' to False to
inhibit setting the prompt to the UNIQUE_PROMPT. Remember that pxssh
uses a unique prompt in the prompt() method. If the original prompt is
not reset then this will disable the prompt() method unless you
manually set the PROMPT attribute. """
ssh_options = '-q'
if self.force_password:
ssh_options = ssh_options + ' ' + self.SSH_OPTS
if port is not None:
ssh_options = ssh_options + ' -p %s'%(str(port))
cmd = "ssh %s -l %s %s" % (ssh_options, username, server)
# This does not distinguish between a remote server 'password' prompt
# and a local ssh 'passphrase' prompt (for unlocking a private key).
spawn._spawn(self, cmd)
i = self.expect(["(?i)are you sure you want to continue connecting", original_prompt, "(?i)(?:password)|(?:passphrase for key)", "(?i)permission denied", "(?i)terminal type", TIMEOUT, "(?i)connection closed by remote host"], timeout=login_timeout)
# First phase
if i==0:
# New certificate -- always accept it.
# This is what you get if SSH does not have the remote host's
# public key stored in the 'known_hosts' cache.
self.sendline("yes")
i = self.expect(["(?i)are you sure you want to continue connecting", original_prompt, "(?i)(?:password)|(?:passphrase for key)", "(?i)permission denied", "(?i)terminal type", TIMEOUT])
if i==2: # password or passphrase
self.sendline(password)
i = self.expect(["(?i)are you sure you want to continue connecting", original_prompt, "(?i)(?:password)|(?:passphrase for key)", "(?i)permission denied", "(?i)terminal type", TIMEOUT])
if i==4:
self.sendline(terminal_type)
i = self.expect(["(?i)are you sure you want to continue connecting", original_prompt, "(?i)(?:password)|(?:passphrase for key)", "(?i)permission denied", "(?i)terminal type", TIMEOUT])
# Second phase
if i==0:
# This is weird. This should not happen twice in a row.
self.close()
raise ExceptionPxssh ('Weird error. Got "are you sure" prompt twice.')
elif i==1: # can occur if you have a public key pair set to authenticate.
### TODO: May NOT be OK if expect() got tricked and matched a false prompt.
pass
elif i==2: # password prompt again
# For incorrect passwords, some ssh servers will
# ask for the password again, others return 'denied' right away.
# If we get the password prompt again then this means
# we didn't get the password right the first time.
self.close()
raise ExceptionPxssh ('password refused')
elif i==3: # permission denied -- password was bad.
self.close()
raise ExceptionPxssh ('permission denied')
elif i==4: # terminal type again? WTF?
self.close()
raise ExceptionPxssh ('Weird error. Got "terminal type" prompt twice.')
elif i==5: # Timeout
#This is tricky... I presume that we are at the command-line prompt.
#It may be that the shell prompt was so weird that we couldn't match
#it. Or it may be that we couldn't log in for some other reason. I
#can't be sure, but it's safe to guess that we did login because if
#I presume wrong and we are not logged in then this should be caught
#later when I try to set the shell prompt.
pass
elif i==6: # Connection closed by remote host
self.close()
raise ExceptionPxssh ('connection closed')
else: # Unexpected
self.close()
raise ExceptionPxssh ('unexpected login response')
if not self.sync_original_prompt():
self.close()
raise ExceptionPxssh ('could not synchronize with original prompt')
# We appear to be in.
# set shell prompt to something unique.
if auto_prompt_reset:
if not self.set_unique_prompt():
self.close()
raise ExceptionPxssh ('could not set shell prompt\n'+self.before)
return True
def logout (self):
"""This sends exit to the remote shell. If there are stopped jobs then
this automatically sends exit twice. """
self.sendline("exit")
index = self.expect([EOF, "(?i)there are stopped jobs"])
if index==1:
self.sendline("exit")
self.expect(EOF)
self.close()
def prompt (self, timeout=20):
"""This matches the shell prompt. This is little more than a short-cut
to the expect() method. This returns True if the shell prompt was
matched. This returns False if there was a timeout. Note that if you
called login() with auto_prompt_reset set to False then you should have
manually set the PROMPT attribute to a regex pattern for matching the
prompt. """
i = self.expect([self.PROMPT, TIMEOUT], timeout=timeout)
if i==1:
return False
return True
def set_unique_prompt (self):
"""This sets the remote prompt to something more unique than # or $.
This makes it easier for the prompt() method to match the shell prompt
unambiguously. This method is called automatically by the login()
method, but you may want to call it manually if you somehow reset the
shell prompt. For example, if you 'su' to a different user then you
will need to manually reset the prompt. This sends shell commands to
the remote host to set the prompt, so this assumes the remote host is
ready to receive commands.
Alternatively, you may use your own prompt pattern. Just set the PROMPT
attribute to a regular expression that matches it. In this case you
should call login() with auto_prompt_reset=False; then set the PROMPT
attribute. After that the prompt() method will try to match your prompt
pattern."""
self.sendline ("unset PROMPT_COMMAND")
self.sendline (self.PROMPT_SET_SH) # sh-style
i = self.expect ([TIMEOUT, self.PROMPT], timeout=10)
if i == 0: # csh-style
self.sendline (self.PROMPT_SET_CSH)
i = self.expect ([TIMEOUT, self.PROMPT], timeout=10)
if i == 0:
return False
return True
# vi:ts=4:sw=4:expandtab:ft=python:

View File

@ -0,0 +1,380 @@
"""This implements a virtual screen. This is used to support ANSI terminal
emulation. The screen representation and state is implemented in this class.
Most of the methods are inspired by ANSI screen control codes. The ANSI class
extends this class to add parsing of ANSI escape codes.
$Id: screen.py 486 2007-07-13 01:04:16Z noah $
"""
import copy
NUL = 0 # Fill character; ignored on input.
ENQ = 5 # Transmit answerback message.
BEL = 7 # Ring the bell.
BS = 8 # Move cursor left.
HT = 9 # Move cursor to next tab stop.
LF = 10 # Line feed.
VT = 11 # Same as LF.
FF = 12 # Same as LF.
CR = 13 # Move cursor to left margin or newline.
SO = 14 # Invoke G1 character set.
SI = 15 # Invoke G0 character set.
XON = 17 # Resume transmission.
XOFF = 19 # Halt transmission.
CAN = 24 # Cancel escape sequence.
SUB = 26 # Same as CAN.
ESC = 27 # Introduce a control sequence.
DEL = 127 # Fill character; ignored on input.
SPACE = chr(32) # Space or blank character.
def constrain (n, min, max):
"""This returns a number, n constrained to the min and max bounds. """
if n < min:
return min
if n > max:
return max
return n
class screen:
"""This object maintains the state of a virtual text screen as a
rectangluar array. This maintains a virtual cursor position and handles
scrolling as characters are added. This supports most of the methods needed
by an ANSI text screen. Row and column indexes are 1-based (not zero-based,
like arrays). """
def __init__ (self, r=24,c=80):
"""This initializes a blank scree of the given dimentions."""
self.rows = r
self.cols = c
self.cur_r = 1
self.cur_c = 1
self.cur_saved_r = 1
self.cur_saved_c = 1
self.scroll_row_start = 1
self.scroll_row_end = self.rows
self.w = [ [SPACE] * self.cols for c in range(self.rows)]
def __str__ (self):
"""This returns a printable representation of the screen. The end of
each screen line is terminated by a newline. """
return '\n'.join ([ ''.join(c) for c in self.w ])
def dump (self):
"""This returns a copy of the screen as a string. This is similar to
__str__ except that lines are not terminated with line feeds. """
return ''.join ([ ''.join(c) for c in self.w ])
def pretty (self):
"""This returns a copy of the screen as a string with an ASCII text box
around the screen border. This is similar to __str__ except that it
adds a box. """
top_bot = '+' + '-'*self.cols + '+\n'
return top_bot + '\n'.join(['|'+line+'|' for line in str(self).split('\n')]) + '\n' + top_bot
def fill (self, ch=SPACE):
self.fill_region (1,1,self.rows,self.cols, ch)
def fill_region (self, rs,cs, re,ce, ch=SPACE):
rs = constrain (rs, 1, self.rows)
re = constrain (re, 1, self.rows)
cs = constrain (cs, 1, self.cols)
ce = constrain (ce, 1, self.cols)
if rs > re:
rs, re = re, rs
if cs > ce:
cs, ce = ce, cs
for r in range (rs, re+1):
for c in range (cs, ce + 1):
self.put_abs (r,c,ch)
def cr (self):
"""This moves the cursor to the beginning (col 1) of the current row.
"""
self.cursor_home (self.cur_r, 1)
def lf (self):
"""This moves the cursor down with scrolling.
"""
old_r = self.cur_r
self.cursor_down()
if old_r == self.cur_r:
self.scroll_up ()
self.erase_line()
def crlf (self):
"""This advances the cursor with CRLF properties.
The cursor will line wrap and the screen may scroll.
"""
self.cr ()
self.lf ()
def newline (self):
"""This is an alias for crlf().
"""
self.crlf()
def put_abs (self, r, c, ch):
"""Screen array starts at 1 index."""
r = constrain (r, 1, self.rows)
c = constrain (c, 1, self.cols)
ch = str(ch)[0]
self.w[r-1][c-1] = ch
def put (self, ch):
"""This puts a characters at the current cursor position.
"""
self.put_abs (self.cur_r, self.cur_c, ch)
def insert_abs (self, r, c, ch):
"""This inserts a character at (r,c). Everything under
and to the right is shifted right one character.
The last character of the line is lost.
"""
r = constrain (r, 1, self.rows)
c = constrain (c, 1, self.cols)
for ci in range (self.cols, c, -1):
self.put_abs (r,ci, self.get_abs(r,ci-1))
self.put_abs (r,c,ch)
def insert (self, ch):
self.insert_abs (self.cur_r, self.cur_c, ch)
def get_abs (self, r, c):
r = constrain (r, 1, self.rows)
c = constrain (c, 1, self.cols)
return self.w[r-1][c-1]
def get (self):
self.get_abs (self.cur_r, self.cur_c)
def get_region (self, rs,cs, re,ce):
"""This returns a list of lines representing the region.
"""
rs = constrain (rs, 1, self.rows)
re = constrain (re, 1, self.rows)
cs = constrain (cs, 1, self.cols)
ce = constrain (ce, 1, self.cols)
if rs > re:
rs, re = re, rs
if cs > ce:
cs, ce = ce, cs
sc = []
for r in range (rs, re+1):
line = ''
for c in range (cs, ce + 1):
ch = self.get_abs (r,c)
line = line + ch
sc.append (line)
return sc
def cursor_constrain (self):
"""This keeps the cursor within the screen area.
"""
self.cur_r = constrain (self.cur_r, 1, self.rows)
self.cur_c = constrain (self.cur_c, 1, self.cols)
def cursor_home (self, r=1, c=1): # <ESC>[{ROW};{COLUMN}H
self.cur_r = r
self.cur_c = c
self.cursor_constrain ()
def cursor_back (self,count=1): # <ESC>[{COUNT}D (not confused with down)
self.cur_c = self.cur_c - count
self.cursor_constrain ()
def cursor_down (self,count=1): # <ESC>[{COUNT}B (not confused with back)
self.cur_r = self.cur_r + count
self.cursor_constrain ()
def cursor_forward (self,count=1): # <ESC>[{COUNT}C
self.cur_c = self.cur_c + count
self.cursor_constrain ()
def cursor_up (self,count=1): # <ESC>[{COUNT}A
self.cur_r = self.cur_r - count
self.cursor_constrain ()
def cursor_up_reverse (self): # <ESC> M (called RI -- Reverse Index)
old_r = self.cur_r
self.cursor_up()
if old_r == self.cur_r:
self.scroll_up()
def cursor_force_position (self, r, c): # <ESC>[{ROW};{COLUMN}f
"""Identical to Cursor Home."""
self.cursor_home (r, c)
def cursor_save (self): # <ESC>[s
"""Save current cursor position."""
self.cursor_save_attrs()
def cursor_unsave (self): # <ESC>[u
"""Restores cursor position after a Save Cursor."""
self.cursor_restore_attrs()
def cursor_save_attrs (self): # <ESC>7
"""Save current cursor position."""
self.cur_saved_r = self.cur_r
self.cur_saved_c = self.cur_c
def cursor_restore_attrs (self): # <ESC>8
"""Restores cursor position after a Save Cursor."""
self.cursor_home (self.cur_saved_r, self.cur_saved_c)
def scroll_constrain (self):
"""This keeps the scroll region within the screen region."""
if self.scroll_row_start <= 0:
self.scroll_row_start = 1
if self.scroll_row_end > self.rows:
self.scroll_row_end = self.rows
def scroll_screen (self): # <ESC>[r
"""Enable scrolling for entire display."""
self.scroll_row_start = 1
self.scroll_row_end = self.rows
def scroll_screen_rows (self, rs, re): # <ESC>[{start};{end}r
"""Enable scrolling from row {start} to row {end}."""
self.scroll_row_start = rs
self.scroll_row_end = re
self.scroll_constrain()
def scroll_down (self): # <ESC>D
"""Scroll display down one line."""
# Screen is indexed from 1, but arrays are indexed from 0.
s = self.scroll_row_start - 1
e = self.scroll_row_end - 1
self.w[s+1:e+1] = copy.deepcopy(self.w[s:e])
def scroll_up (self): # <ESC>M
"""Scroll display up one line."""
# Screen is indexed from 1, but arrays are indexed from 0.
s = self.scroll_row_start - 1
e = self.scroll_row_end - 1
self.w[s:e] = copy.deepcopy(self.w[s+1:e+1])
def erase_end_of_line (self): # <ESC>[0K -or- <ESC>[K
"""Erases from the current cursor position to the end of the current
line."""
self.fill_region (self.cur_r, self.cur_c, self.cur_r, self.cols)
def erase_start_of_line (self): # <ESC>[1K
"""Erases from the current cursor position to the start of the current
line."""
self.fill_region (self.cur_r, 1, self.cur_r, self.cur_c)
def erase_line (self): # <ESC>[2K
"""Erases the entire current line."""
self.fill_region (self.cur_r, 1, self.cur_r, self.cols)
def erase_down (self): # <ESC>[0J -or- <ESC>[J
"""Erases the screen from the current line down to the bottom of the
screen."""
self.erase_end_of_line ()
self.fill_region (self.cur_r + 1, 1, self.rows, self.cols)
def erase_up (self): # <ESC>[1J
"""Erases the screen from the current line up to the top of the
screen."""
self.erase_start_of_line ()
self.fill_region (self.cur_r-1, 1, 1, self.cols)
def erase_screen (self): # <ESC>[2J
"""Erases the screen with the background color."""
self.fill ()
def set_tab (self): # <ESC>H
"""Sets a tab at the current position."""
pass
def clear_tab (self): # <ESC>[g
"""Clears tab at the current position."""
pass
def clear_all_tabs (self): # <ESC>[3g
"""Clears all tabs."""
pass
# Insert line Esc [ Pn L
# Delete line Esc [ Pn M
# Delete character Esc [ Pn P
# Scrolling region Esc [ Pn(top);Pn(bot) r

39
lldb/test/pexpect-2.4/setup.py Executable file
View File

@ -0,0 +1,39 @@
'''
$Revision: 485 $
$Date: 2007-07-12 15:23:15 -0700 (Thu, 12 Jul 2007) $
'''
from distutils.core import setup
setup (name='pexpect',
version='2.4',
py_modules=['pexpect', 'pxssh', 'fdpexpect', 'FSM', 'screen', 'ANSI'],
description='Pexpect is a pure Python Expect. It allows easy control of other applications.',
author='Noah Spurrier',
author_email='noah@noah.org',
url='http://pexpect.sourceforge.net/',
license='MIT license',
platforms='UNIX',
)
# classifiers = [
# 'Development Status :: 4 - Beta',
# 'Environment :: Console',
# 'Environment :: Console (Text Based)',
# 'Intended Audience :: Developers',
# 'Intended Audience :: System Administrators',
# 'Intended Audience :: Quality Engineers',
# 'License :: OSI Approved :: Python Software Foundation License',
# 'Operating System :: POSIX',
# 'Operating System :: MacOS :: MacOS X',
# 'Programming Language :: Python',
# 'Topic :: Software Development',
# 'Topic :: Software Development :: Libraries :: Python Modules',
# 'Topic :: Software Development :: Quality Assurance',
# 'Topic :: Software Development :: Testing',
# 'Topic :: System, System :: Archiving :: Packaging, System :: Installation/Setup',
# 'Topic :: System :: Shells',
# 'Topic :: System :: Software Distribution',
# 'Topic :: Terminals, Utilities',
# ],