# Written by Cameron Dale
# see LICENSE.txt for license information
#
# $Id$
"""Listen for download requests from Apt.
@type logger: C{logging.Logger}
@var logger: the logger to send all log messages to for this module
@type alas: C{string}
@var alas: the message to send when the data is not found
@type VERSION: C{string}
@var VERSION: the Server identifier sent to all sites
"""
from DebTorrent.subnetparse import IP_List, ipv6_to_ipv4, is_ipv4
from DebTorrent.iprangeparse import IP_List as IP_Range_List
from DebTorrent.bencode import bencode
from DebTorrent.zurllib import quote, unquote
from Filter import Filter
from urlparse import urlparse
from os.path import join
from cStringIO import StringIO
from time import time, gmtime, strftime
from DebTorrent.clock import clock
from sha import sha
from binascii import b2a_hex, a2b_hex
from makemetafile import TorrentCreator
from DebTorrent.HTTPCache import HTTPCache
import os, logging
from DebTorrent.__init__ import version, product_name,version_short
from debian_bundle import deb822
logger = logging.getLogger('DebTorrent.BT1.AptListener')
VERSION = product_name+'/'+version_short
alas = 'your file may exist elsewhere in the universe\nbut alas, not here\n'
def isotime(secs = None):
"""Create an ISO formatted string of the time.
@type secs: C{float}
@param secs: number of seconds since the epoch
(optional, default is to use the current time)
@rtype: C{string}
@return: the ISO formatted string representation of the time
"""
if secs == None:
secs = time()
return strftime('%Y-%m-%d %H:%M UTC', gmtime(secs))
class AptListener:
"""Listen for Apt requests to download files.
@type handler: L{DebTorrent.launchmanycore.LaunchMany}
@ivar handler: the download handler to use
@type config: C{dictionary}
@ivar config: the configuration parameters
@type parse_ip_files: C{int}
@ivar parse_ip_files: seconds between reloading of the
lists of allowed and banned IPs
@type favicon: C{string}
@ivar favicon: file containing x-icon data
@type rawserver: L{DebTorrent.RawServer}
@ivar rawserver: the server to use for scheduling
@type state: C{dictionary}
@ivar state: the current state information for the tracking
@type allowed_IPs: L{DebTorrent.subnetparse.IP_List}
@ivar allowed_IPs: the IPs that are allowed to connect, or None if all are
@type banned_IPs: L{DebTorrent.iprangeparse.IP_List}
@ivar banned_IPs: the IPs that are not allowed to connect
@type allowed_ip_mtime: C{int}
@ivar allowed_ip_mtime: the last modification time of the allowed IPs file
@type banned_ip_mtime: C{int}
@ivar banned_ip_mtime: the last modification time of the banned IPs file
@type allow_get: C{boolean}
@ivar allow_get: whether downloading of torrent files is allowed
@type uq_broken: C{boolean}
@ivar uq_broken: whether URL quoting of '+' is broken
@type Filter: L{Filter.Filter}
@ivar Filter: not used
@type Cache: L{DebTorrent.HTTPCache.HTTPCache}
@ivar Cache: the cache of downloaded files
@type cache_waiting: C{dictionary}
@ivar cache_waiting: the pending HTTP get requests that are waiting for download
from the cache. Keys are strings that are the path being requested, values
are lists of L{DebTorrent.HTTPHandler.HTTPConnection} objects which are the
requests that are pending for that path.
@type identifiers: C{dictionary}
@ivar identifiers: keys are the files found in Release files, and values
are the identifiers found in the Release file
(currently (C{string}, C{string}), the 'Origin' and 'Label' fields)
@type request_queue: C{dictionary}
@ivar request_queue: the pending HTTP package requests that are waiting for download.
Keys are the file names (including mirror) requested, values are dictionaries
with keys of L{DebTorrent.HTTPHandler.HTTPConnection} objects and values of
(L{DebTorrent.download_bt1.BT1Download}, C{int},
L{DebTorrent.HTTPHandler.HTTPRequest}, C{list} of C{int}, C{float})
which are the torrent downloader, file index, HTTP request object to answer,
list of pieces needed, and the time of the original request.
"""
def __init__(self, handler, config, rawserver):
"""Initialize the instance.
@type handler: L{DebTorrent.launchmanycore.LaunchMany}
@param handler: the download handler to use
@type config: C{dictionary}
@param config: the configuration parameters
@type rawserver: L{DebTorrent.RawServer}
@param rawserver: the server to use for scheduling
"""
self.handler = handler
self.config = config
favicon = config['favicon']
self.parse_ip_files = config['parse_ip_files']
self.favicon = None
if favicon:
try:
h = open(favicon,'r')
self.favicon = h.read()
h.close()
except:
logger.warning('specified favicon file does not exist.')
self.rawserver = rawserver
self.state = {}
self.allowed_IPs = None
self.banned_IPs = None
if config['allowed_ips'] or config['banned_ips']:
self.allowed_ip_mtime = 0
self.banned_ip_mtime = 0
self.read_ip_lists()
self.allow_get = config['allow_get']
self.uq_broken = unquote('+') != ' '
self.Filter = Filter(rawserver.add_task)
if config['download_dir']:
self.Cache = HTTPCache(rawserver, config['download_dir'])
else:
self.Cache = HTTPCache(rawserver, self.handler.configdir.home_dir)
self.cache_waiting = {}
self.identifiers = {}
self.request_queue = {}
rawserver.add_task(self.process_queue, 1)
def enqueue_request(self, connection, file, downloader, file_num, httpreq, pieces_needed):
"""Add a new download request to the queue of those waiting for pieces.
@type connection: L{DebTorrent.HTTPHandler.HTTPConnection}
@param connection: the conection the request came in on
@type file: C{string}
@param file: the file to download, starting with the mirror name
@type downloader: L{DebTorrent.download_bt1.BT1Download}
@param downloader: the torrent download that has the file
@type file_num: C{int}
@param file_num: the index of the file in the torrent
@type httpreq: L{DebTorrent.HTTPHandler.HTTPRequest}
@param httpreq: the HTTP request object to answer (for queueing)
@type pieces_needed: C{list} of C{int}
@param pieces_needed: the list of pieces in the torrent that still
need to download
"""
# Get the file's queue and check it for this connection
queue = self.request_queue.setdefault(file, {})
if connection in queue:
logger.error('Received multiple requests for the same file on one connection')
return
logger.info('queueing request as file '+file+' needs pieces: '+str(pieces_needed))
queue[connection] = (downloader, file_num, httpreq, pieces_needed, clock())
def process_queue(self):
"""Process the queue of waiting requests."""
# Schedule it again
self.rawserver.add_task(self.process_queue, 1)
closed_conns = []
open_conns = {}
for file, queue in self.request_queue.items():
for c, v in queue.items():
# Check for a closed connection
if c.closed:
closed_conns.append((file, c))
logger.warning('connection closed while request queued for file '+file)
continue
# Remove the downloaded pieces from the list of needed ones
piece_removed = False
for piece in list(v[3]):
if v[0].storagewrapper.do_I_have(piece):
logger.debug('queued request for file '+file+' got piece '+str(piece))
v[3].remove(piece)
piece_removed = True
# If no more pieces are needed, return the answer and remove the request
if not v[3]:
logger.info('queued request for file '+file+' is complete')
closed_conns.append((file, c))
v[0].storagewrapper.set_file_readonly(v[1])
self.answer_package(c, file, v[0], v[1], v[2])
continue
# Otherwise send an update message
# Currently disabled due to apt not handling this well
#elif piece_removed:
# self.package_update(c, file, v[0], v[1], v[2], v[3])
open_conns.setdefault(c, {}).setdefault(v[0], []).extend(v[3])
# Remove closed/finished connections from the queue
for (file, c) in closed_conns:
self.request_queue[file].pop(c)
if not self.request_queue[file]:
self.request_queue.pop(file)
self.connection_update(open_conns)
def get_infopage(self, show_piecemaps):
"""Format the info page to display for normal browsers.
Formats the currently downloading torrents into a table in human-readable
format to display in a browser window.
@type show_piecemaps: C{string}
@param show_piecemaps: "svg" to include SVG piecemaps. "png" may be supported in future
@rtype: (C{int}, C{string}, C{dictionary}, C{string})
@return: the HTTP status code, status message, headers, and message body
"""
try:
if not self.config['show_infopage']:
return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
red = self.config['infopage_redirect']
if red:
return (302, 'Found', {'Server': VERSION, 'Content-Type': 'text/html', 'Location': red},
'Click Here')
# Write the document header
s = StringIO()
s.write('\n' \
'DebTorrent download info\n')
if self.favicon is not None:
s.write('\n')
s.write('\n\n' \
'
DebTorrent download info
\n'\
'
\n'
'
client version: %s
\n' \
'
client time: %s
\n' \
'
\n' % (version, isotime()))
# Write the table headers
s.write('
\n' \
'
\n' \
'
name/ \n' \
'info hash
\n' \
'
status
\n' \
'
progress
\n' \
'
peers/ \n' \
'seeds
\n' \
'
distributed copies
\n' \
'
download/ \n' \
'upload
\n' \
'
downloaded (HTTP)/ \n' \
'uploaded
\n' \
'
size
\n' \
'
time remaining
\n' \
'
last error message
\n' \
'
\n')
# Get the data from the statistics gatherer
data = self.handler.gather_stats()
if not data:
s.write('
no torrents
\n')
# Display a table row for each running torrent
for x in data:
( name, hash, status, progress, peers, seeds, seedsmsg, dist,
uprate, dnrate, upamt, dnamt, httpdnamt, size, t, msg ) = x
if self.allow_get:
linkname = '' + name + ''
else:
linkname = name
s.write('
info hash: SHA1 hash of the "info" section of the metainfo (.dtorrent) file
\n' \
'
status: the current operation under way
\n' \
'
progress: the current progress in the operation under way
\n' \
'
distributed copies: the number of copies of the complete torrent seen in non-seeding peers
\n' \
'
\n')
# Draw the piece maps
s.write('
Piece maps
\n')
if show_piecemaps == 'svg':
for x in data:
( name, hash, status, progress, peers, seeds, seedsmsg, dist,
uprate, dnrate, upamt, dnamt, httpdnamt, size, t, msg ) = x
s.write('
%s (%s)
\n' % (name, b2a_hex(hash)))
s.write('\n'
% (b2a_hex(hash), b2a_hex(hash), self.svg_height_piecemap(peers)))
#TODO use an accurate size for the image
s.write('
\n' \
'
blue: You have this piece
\n' \
'
purple: A peer has this piece
\n' \
'
white: You/Peer does not have this piece, but does have a later piece
\n' \
'
grey: You/Peer does not have this piece, and does not have any later piece
')
s.write('\n' \
'\n')
return (200, 'OK', {'Server': VERSION, 'Content-Type': 'text/html; charset=iso-8859-1'}, s.getvalue())
except:
logger.exception('Error returning info_page')
return (500, 'Internal Server Error', {'Server': VERSION, 'Content-Type': 'text/html; charset=iso-8859-1'}, 'Server Error')
def svg_height_piecemap(self, number_of_peers):
"""Returns the size (currently only height) of the piecemap image.
This has an inbuild race condition - the actual request to draw the piecemap may come in a separate request.
"""
#TODO - move the SVG stuff to a separate class, make these class constants
markwidth=1
markheight=6
linepadding=1
lineheight=markheight + linepadding
# This works round the behaviour of Iceweasel, which stretches these images to be a minimum of 50 pixels hight
totalheight = lineheight * (number_of_peers+1) + linepadding
workaroundheight = totalheight
if (workaroundheight < 50):
workaroundheight = 50
return workaroundheight
def get_piecemap_for_torrent_svg(self, info_hash):
"""Shows the piecemap (which peers have what) for a given torrent.
The HTTP request should include the appropriate torrent hash ( http://.../piecemap.svg?info_hash=... )
@type info_hash: C{string}
@param info_hash: the info_hash passed to the HTTP query
@rtype: (C{int}, C{string}, C{dictionary}, C{string})
@return: the HTTP status code, status message, headers, and message body
"""
try:
if not self.config['show_infopage']:
return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
if info_hash == None:
return (400, 'Bad request', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, 'No hash specified in request')
hash = a2b_hex(info_hash)
piecelist_list = self.handler.get_piecemap(hash)
if piecelist_list == None:
return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, 'Unknown hash')
s = StringIO()
self.svg_draw_piecemap(s, piecelist_list, ["blue", "purple"])
return (200, 'OK', {'Server': VERSION, 'Content-Type': 'image/svg+xml; charset=utf-8'}, s.getvalue())
except:
logger.exception('Error returning info_page')
return (500, 'Internal Server Error', {'Server': VERSION, 'Content-Type': 'text/html; charset=iso-8859-1'}, 'Server Error')
def svg_draw_piecemap(self, s, piecelist_list, color_list):
"""Draw (in SVG) a map of which pieces are present in the piecelist
@type s: C{StringIO}
@param s: Output stream for data that will be sent to the webbrowser.
@type piecelist_list: C{List} of C{Bitfield}
@param piecelist_list: The bitfields representing which pieces are present.
@type color_list: C{List} of C{String}
@param color_list: List of colors for drawing the pieces. color_list[i] is used for piecelist_list[i]; if there are more piecelists than colors then the last color is reused.
"""
# Constants for the size of the drawing
# Vertically, each line is of total height lineheight, with padding at the bottom.
markwidth=1
markheight=6
linepadding=1
lineheight=markheight + linepadding
fittowindow = True
# Since some piecelists have more pieces than others, find the longest.
mostpieces = 0
for piecelist in piecelist_list:
if (len(piecelist) > mostpieces):
mostpieces = len(piecelist)
# The bitfields have a lot of whitespace. Compress the drawing by showing only the pieces held by at least one peer.
# piecelist[piecenumber] will be drawn at pixellist[piecenumber]
pixellist = [0] * mostpieces
y = 0
for x in xrange(0, mostpieces):
someonehas = False
pixellist[x] = y
for i in xrange(0, len(piecelist_list)):
if (piecelist_list[i][x]):
someonehas = True
if someonehas:
y += 1
# And this is the width of the compressed drawing
imagewidth = y
# This works round the behaviour of Iceweasel, which stretches these images to be a minimum of 50 pixels hight
totalheight = lineheight * len(piecelist_list) + linepadding
workaroundheight = totalheight
if (workaroundheight < 50):
workaroundheight = 50
if (fittowindow):
s.write('\n'
% (workaroundheight, markwidth * imagewidth, workaroundheight))
else:
s.write('\n'
% (markwidth * imagewidth, workaroundheight))
# Draw the background in one go.
s.write('\n'
% (markwidth * imagewidth, totalheight))
color = "blue"; # Default which should be overridden by color_list[0]
for i in xrange(0, len(piecelist_list)):
piecelist = piecelist_list[i]
y = i * lineheight + linepadding # the y-coordinate that we're drawing at
# This makes each row white, on a black grid (the previous background showing through)
s.write('\n'
% (y, imagewidth * markwidth, markheight))
# Set default drawing color
if (i < len(color_list)):
color = color_list[i]
s.write('\n' % (color))
# Clump the data together into runs of "we have this piece" and "we lack this piece"
runstart = 0
runtype = piecelist[0]
for piece in xrange (1, len(piecelist)):
if (piecelist[piece] != runtype):
# Just past the end of the current run. Draw it if it was a "we have" run.
if (runtype):
s.write('\n'
% (pixellist[runstart] * markwidth, y, (pixellist[piece]-pixellist[runstart]) * markwidth, markheight,
runstart, piece-1))
# Now start counting the new run.
runstart = piece
runtype = piecelist[piece]
# Reached the end of the data, draw the final run
if (runtype):
s.write('\n'
% (pixellist[runstart] * markwidth, y, (pixellist[piece]-pixellist[runstart]) * markwidth, markheight,
runstart, piece-1))
else:
# Show how far the "no pieces" bit is
s.write('\n'
% (pixellist[runstart] * markwidth, y, (pixellist[piece]-pixellist[runstart]) * markwidth, markheight))
s.write("\n\n")
s.write('')
def get_meow(self):
return (200, 'OK', {'Server': VERSION, 'Content-Type': 'text/html; charset=iso-8859-1'}, """\nMeow\n\n\n
\n\n""")
def get_cached(self, connection, path, headers, httpreq):
"""Proxy the (possibly cached) download of a file from a mirror.
@type connection: L{DebTorrent.HTTPHandler.HTTPConnection}
@param connection: the conection the request came in on
@type path: C{list} of C{string}
@param path: the path of the file to download, starting with the mirror name
@type headers: C{dictionary}
@param headers: the headers from the request
@type httpreq: L{DebTorrent.HTTPHandler.HTTPRequest}
@param httpreq: the HTTP request object to answer (for queueing)
@rtype: (C{int}, C{string}, C{dictionary}, C{string})
@return: the HTTP status code, status message, headers, and downloaded file
(or None if the file is being downloaded)
"""
try:
# Deb files don't need to be checked for staleness
uptodate = True
if path[-1][-4:] == '.deb':
uptodate = False
# First check the cache for the file
r, filename = self.Cache.cache_get(path, uptodate, headers.get('if-modified-since', ''))
# If the cache doesn't have it
if not filename:
# Get Debs from the debtorrent download, others are straight download
if path[-1][-4:] == '.deb':
return self.get_package(connection, path, httpreq)
else:
decompress = False
if path[-1] in ('Packages.gz', 'Packages.bz2'):
decompress = True
# Save the connection info and start downloading the file
self.cache_waiting.setdefault('/'.join(path), []).append((connection, httpreq))
self.Cache.download_get(path, self.get_cached_callback, decompress)
return None
# Save the identifiers from Release files
if path[-1] == 'Release':
self.save_identifiers(path, filename)
if path[-1] in ('Packages', 'Packages.gz', 'Packages.bz2'):
TorrentCreator(path, filename, self.start_torrent,
self.rawserver.add_task, self.Cache, self.config,
self.identifiers.get('/'.join(path), None))
# Returning a file, so open the file to be returned
if r[0] != 304:
r = r[0:3] + (open(filename, 'rb'), )
return r
except IOError, e:
logger.exception('While retrieving cache file: ' + '/'.join(path))
try:
(msg, status) = e
except:
status = 404
msg = 'Unknown error occurred'
return (status, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, msg)
def get_cached_callback(self, path, r, filename):
"""Return the newly cached file to the waiting connection.
@type path: C{list} of C{string}
@param path: the path of the file to download, starting with the mirror name
@type r: (C{int}, C{string}, C{dictionary}, C{string})
@param r: the HTTP status code, status message, headers, and cached data
@type filename: C{string}
@param filename: the file containing the successfully downloaded file
"""
# Get the list of connections waiting for this file
connections = self.cache_waiting.pop('/'.join(path), None)
if connections is None:
logger.warning('no connection exists to return the cached file on')
return
# Save the identifiers from Release files
if r[0] in [200, 304] and path[-1] == 'Release':
self.save_identifiers(path, filename)
# If it's a torrent file, start it
if r[0] in [200, 304] and path[-1] in ('Packages', 'Packages.gz', 'Packages.bz2'):
TorrentCreator(path, filename, self.start_torrent,
self.rawserver.add_task, self.Cache, self.config,
self.identifiers.get('/'.join(path), None))
if filename and r[0] != 304:
# Returning a file, so open the file to be returned
r = r[0:3] + (open(filename, 'rb'), )
for (connection, httpreq) in connections:
# Check to make sure the requester is still waiting
if connection.closed:
logger.warning('Retrieved the file, but the requester is gone: '+'/'.join(path))
continue
connection.answer(r, httpreq)
def save_identifiers(self, path, filename):
"""Save the identifiers from a Release file for later downloads.
@type path: C{list} of C{string}
@param path: the path of the Release file to download, starting with the mirror name
@type filename: C{string}
@param filename: the file containing the successfully downloaded Release file
"""
f = open(filename, 'r')
rel = deb822.Release(f)
identifier = (rel.get('Origin', ''), rel.get('Label', ''))
# Read the Packages file names
for file in rel.get('MD5Sum', []):
self.identifiers['/'.join(path[:-1]) + '/' + file['name']] = identifier
for file in rel.get('SHA1', []):
self.identifiers['/'.join(path[:-1]) + '/' + file['name']] = identifier
for file in rel.get('SHA256', []):
self.identifiers['/'.join(path[:-1]) + '/' + file['name']] = identifier
f.close()
def get_package(self, connection, path, httpreq):
"""Download a package file from a torrent.
@type connection: L{DebTorrent.HTTPHandler.HTTPConnection}
@param connection: the conection the request came in on
@type path: C{list} of C{string}
@param path: the path of the file to download, starting with the mirror name
@type httpreq: L{DebTorrent.HTTPHandler.HTTPRequest}
@param httpreq: the HTTP request object to answer (for queueing)
@rtype: (C{int}, C{string}, C{dictionary}, C{string})
@return: the HTTP status code, status message, headers, and package data
(or None if the package is to be downloaded)
"""
# Find the file in one of the torrent downloads
d, f = self.handler.find_file(path[0], path[1:])
if d is None:
logger.warning('Unable to find the file in any torrents: '+'/'.join(path))
return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
# Check if the torrent is running
if d.doneflag.isSet():
logger.error('The needed torrent is not running')
return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
# Check if the file has already been downloaded
pieces_needed = []
start_piece, end_piece = d.fileselector.storage.file_pieces[f]
for piece in xrange(start_piece, end_piece+1):
if not d.storagewrapper.do_I_have(piece):
pieces_needed.append(piece)
if not pieces_needed:
filename = d.getFilepath(f)
file = open(filename, 'rb')
return (200, 'OK', {'Server': VERSION, 'Content-Type': 'text/plain'}, file)
if not d.unpauseflag.isSet():
d.Unpause()
# Enable the download of the piece
d.fileselector.set_priority(f, 1)
# Add the connection to the list of those needing responses
self.enqueue_request(connection, '/'.join(path), d, f, httpreq, pieces_needed)
return None
def connection_update(self, connections):
"""Send a status 103 update message to the open requesters.
The input dictionary has keys which are HTTP connections
(L{DebTorrent.HTTPHandler.HTTPConnection}), while the values are
dictionaries with keys of downloaders
(L{DebTorrent.download_bt1.BT1Download}) and values of the list of
pieces still needed for that connection from that downloader
(C{list} of C{int}).
@type connections: C{dictionary} of L{DebTorrent.HTTPHandler.HTTPConnection}
@param connections: the conections to send updates to
"""
for connection in connections:
# Check to make sure the requester is still waiting
if connection.closed:
return
file_needed = 0L
for d in connections[connection]:
logger.debug('pieces needed: '+ str(connections[connection][d]))
processed_piece = []
for piece in connections[connection][d]:
if piece not in processed_piece:
file_needed += d.storagewrapper.piece_sizes[piece]
processed_piece.append(piece)
message = size_format(file_needed) + ' left'
logger.debug('file needed: '+str(file_needed)+' from '+ str(processed_piece))
# Get the data from the statistics gatherer
data = self.handler.gather_stats()
if not data:
message += ' no torrents'
else:
# Display a table row for each running torrent
total_size = 0L
total_dnrate = 0
for x in data:
( name, hash, status, progress, peers, seeds, seedsmsg, dist,
uprate, dnrate, upamt, dnamt, httpdnamt, size, t, msg ) = x
total_size += size
total_dnrate += dnrate
message += ' at ' + size_format(total_dnrate) + '/s'
if total_dnrate > 0:
message += ' (' + hours(file_needed/total_dnrate) + ')'
connection.answer((103, 'Status Update', {'Server': VERSION, 'Content-Type': 'text/plain',
'Message': 'DebTorrent: ' + message}, ''), None)
def package_update(self, connection, file, d, f, httpreq, pieces_needed):
"""Send a status 102 update message to the requester.
@type connection: L{DebTorrent.HTTPHandler.HTTPConnection}
@param connection: the conection the request came in on
@type file: C{string}
@param file: the file to download, starting with the mirror name
@type d: L{DebTorrent.download_bt1.BT1Download}
@param d: the torrent download that has the file
@type f: C{int}
@param f: the index of the file in the torrent
@type httpreq: L{DebTorrent.HTTPHandler.HTTPRequest}
@param httpreq: the HTTP request object to answer (for queueing)
@type pieces_needed: C{list} of C{int}
@param pieces_needed: the list of pieces in the torrent that still
need to be downloaded
"""
# Check to make sure the requester is still waiting
if connection.closed:
return
# Check if the file has been downloaded
file_needed = 0L
filename, length = d.fileselector.storage.files[f]
for piece in pieces_needed:
file_needed += d.storagewrapper.piece_sizes[piece]
piece_string = 'bytes ' + str(length - file_needed) + '/' + str(length)
connection.answer((102, 'Size Update', {'Server': VERSION, 'Content-Type': 'text/plain',
'Pieces-Downloaded': piece_string}, ''), httpreq)
def answer_package(self, connection, file, d, f, httpreq):
"""Send the newly downloaded package file to the requester.
@type connection: L{DebTorrent.HTTPHandler.HTTPConnection}
@param connection: the conection the request came in on
@type file: C{string}
@param file: the file to download, starting with the mirror name
@type d: L{DebTorrent.download_bt1.BT1Download}
@param d: the torrent download that has the file
@type f: C{int}
@param f: the index of the file in the torrent
@type httpreq: L{DebTorrent.HTTPHandler.HTTPRequest}
@param httpreq: the HTTP request object to answer (for queueing)
"""
# Check to make sure the requester is still waiting
if connection.closed:
return
# Check if the file has been downloaded
pieces_needed = []
start_piece, end_piece = d.fileselector.storage.file_pieces[f]
for piece in xrange(start_piece, end_piece+1):
if not d.storagewrapper.do_I_have(piece):
pieces_needed.append(piece)
if not pieces_needed:
filename = d.getFilepath(f)
file = open(filename, 'rb')
connection.answer((200, 'OK', {'Server': VERSION, 'Content-Type': 'text/plain'}, file), httpreq)
return
# Something strange has happened, requeue it
logger.warning('requeuing request for file '+str(f)+' as it still needs pieces: '+str(pieces_needed))
self.enqueue_request(connection, file, d, f, httpreq, pieces_needed)
def start_torrent(self, response, name, path):
infohash = sha(bencode(response['info'])).digest()
a = {}
a['path'] = '/'.join(path)
a['file'] = name
a['type'] = path[-1]
mirror = path[0]
for i in path[1:path.index('dists')]:
mirror = join(mirror, i)
a['mirror'] = mirror
i = response['info']
l = 0
nf = 0
if i.has_key('length'):
l = i.get('length',0)
nf = 1
elif i.has_key('files'):
for li in i['files']:
nf += 1
if li.has_key('length'):
l += li['length']
a['numfiles'] = nf
a['length'] = l
a['name'] = name
a['time'] = self.Cache.get_file_mtime(path)
def setkey(k, d = response, a = a):
if d.has_key(k):
a[k] = d[k]
setkey('failure reason')
setkey('warning message')
setkey('announce-list')
a['metainfo'] = response
logger.info('Adding torrent '+name+': '+b2a_hex(infohash))
self.handler.add(infohash, a)
def get_file(self, hash):
"""Get the metainfo file for a torrent.
@type hash: C{string}
@param hash: the infohash of the torrent to get the metainfo of
@rtype: (C{int}, C{string}, C{dictionary}, C{string})
@return: the HTTP status code, status message, headers, and bencoded
metainfo file
"""
if not self.allow_get:
logger.warning('Unauthorized request for torrent file: '+b2a_hex(hash))
return (400, 'Not Authorized', {'Server': VERSION,
'Content-Type': 'text/plain',
'Pragma': 'no-cache'},
'get function is not available with this tracker.')
if hash not in self.handler.torrent_cache:
logger.warning('Request for unknown torrent file: '+b2a_hex(hash))
return (404, 'Not Found', {'Server': VERSION,
'Content-Type': 'text/plain',
'Pragma': 'no-cache'},
alas)
fname = self.handler.torrent_cache[hash]['file'] + '.dtorrent'
response = self.handler.configdir.getTorrentFile(hash)
if not response:
logger.warning('Could not retrieve torrent file: '+b2a_hex(hash))
return (404, 'Not Found', {'Server': VERSION,
'Content-Type': 'text/plain',
'Pragma': 'no-cache'},
alas)
return (200, 'OK', {'Server': VERSION,
'Content-Type': 'application/x-debtorrent',
'Content-Disposition': 'attachment; filename=' + fname},
response)
def get(self, connection, path, headers, httpreq):
"""Respond to a GET request.
Process a GET request from APT/browser/other. Process the request,
calling the helper functions above if needed. Return the response to
be returned to the requester.
@type connection: L{DebTorrent.HTTPHandler.HTTPConnection}
@param connection: the conection the request came in on
@type path: C{string}
@param path: the URL being requested
@type headers: C{dictionary}
@param headers: the headers from the request
@type httpreq: L{DebTorrent.HTTPHandler.HTTPRequest}
@param httpreq: the HTTP request object to answer (for queueing)
@rtype: (C{int}, C{string}, C{dictionary}, C{string})
@return: the HTTP status code, status message, headers, and message body
"""
real_ip = connection.get_ip()
ip = real_ip
if not is_ipv4(ip):
try:
ip = ipv6_to_ipv4(ip)
except ValueError:
pass
if ( (self.allowed_IPs and not self.allowed_IPs.includes(ip))
or (self.banned_IPs and self.banned_IPs.includes(ip)) ):
logger.warning('Unauthorized request from '+ip+' for: '+path)
return (400, 'Not Authorized', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'},
bencode({'failure reason':
'your IP is not allowed on this proxy'}))
paramslist = {}
def params(key, default = None, l = paramslist):
"""Get the user parameter, or the default.
@type key: C{string}
@param key: the parameter to get
@type default: C{string}
@param default: the default value to use if no parameter is set
(optional, defaults to None)
@type l: C{dictionary}
@param l: the user parameters (optional, defaults to L{paramslist})
@rtype: C{string}
@return: the parameter's value
"""
if l.has_key(key):
return l[key][0]
return default
try:
(scheme, netloc, path, pars, query, fragment) = urlparse(path)
if self.uq_broken == 1:
#path = path.replace('+',' ') What is this!
query = query.replace('+',' ')
path = unquote(path)[1:]
for s in query.split('&'):
if s:
i = s.index('=')
kw = unquote(s[:i])
paramslist.setdefault(kw, [])
paramslist[kw] += [unquote(s[i+1:])]
logger.debug('paramslist['+str(kw)+'] =='+str(paramslist[kw]))
if path == '' or path == 'index.html':
return self.get_infopage(params('piecemaps'))
if path == 'piecemap.svg':
return self.get_piecemap_for_torrent_svg(params('info_hash'))
if path == 'meow':
return self.get_meow()
if path == 'favicon.ico':
if self.favicon is not None:
return (200, 'OK', {'Server': VERSION, 'Content-Type' : 'image/x-icon'}, self.favicon)
else:
return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
if (path == 'file'):
return self.get_file(params('info_hash'))
# Process the rest as a proxy
path = path.split('/')
if 'Packages.diff' in path:
return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
return self.get_cached(connection, path, headers, httpreq)
except ValueError, e:
logger.exception('Bad request from: '+ip)
return (400, 'Bad Request', {'Server': VERSION, 'Content-Type': 'text/plain'},
'you sent me garbage - ' + str(e))
def read_ip_lists(self):
"""Periodically parse the allowed and banned IPs lists."""
self.rawserver.add_task(self.read_ip_lists,self.parse_ip_files)
f = self.config['allowed_ips']
if f and self.allowed_ip_mtime != os.path.getmtime(f):
self.allowed_IPs = IP_List()
try:
self.allowed_IPs.read_fieldlist(f)
self.allowed_ip_mtime = os.path.getmtime(f)
except (IOError, OSError):
logger.exception('unable to read allowed_IP list')
f = self.config['banned_ips']
if f and self.banned_ip_mtime != os.path.getmtime(f):
self.banned_IPs = IP_Range_List()
try:
self.banned_IPs.read_rangelist(f)
self.banned_ip_mtime = os.path.getmtime(f)
except (IOError, OSError):
logger.exception('unable to read banned_IP list')
def size_format(s):
"""Format a byte size for reading by the user.
@type s: C{long}
@param s: the number of bytes
@rtype: C{string}
@return: the formatted size with appropriate units
"""
if (s < 1024):
r = str(s) + 'B'
elif (s < 10485):
r = str(int((s/1024.0)*100.0)/100.0) + 'KiB'
elif (s < 104857):
r = str(int((s/1024.0)*10.0)/10.0) + 'KiB'
elif (s < 1048576):
r = str(int(s/1024)) + 'KiB'
elif (s < 10737418L):
r = str(int((s/1048576.0)*100.0)/100.0) + 'MiB'
elif (s < 107374182L):
r = str(int((s/1048576.0)*10.0)/10.0) + 'MiB'
elif (s < 1073741824L):
r = str(int(s/1048576)) + 'MiB'
elif (s < 1099511627776L):
r = str(int((s/1073741824.0)*100.0)/100.0) + 'GiB'
else:
r = str(int((s/1099511627776.0)*100.0)/100.0) + 'TiB'
return(r)
def hours(n):
"""Formats seconds into a human-readable time.
Formats a given number of seconds into a human-readable time appropriate
for display to the user.
@type n: C{int}
@param n: the number of seconds
@rtype: C{string}
@return: a displayable representation of the number of seconds
"""
if n == 0:
return 'complete!'
try:
n = int(n)
assert n >= 0 and n < 5184000 # 60 days
except:
return ''
m, s = divmod(n, 60)
h, m = divmod(m, 60)
if h > 0:
return '%dh%02dm%02ds' % (h, m, s)
else:
return '%dm%02ds' % (m, s)
