Browse Source

The BIG refactoring [tm]. Too many changes to count them. If it doesn't suite you, revert to last version.

erdgeist 13 years ago
parent
commit
334c6e4bbb
  1. 7
      Makefile
  2. 42
      opentracker.c
  3. 11
      opentracker.xcodeproj/project.pbxproj
  4. 2
      ot_accesslist.c
  5. 7
      ot_accesslist.h
  6. 148
      ot_clean.c
  7. 8
      ot_clean.h
  8. 1
      ot_fullscrape.c
  9. 62
      ot_http.c
  10. 2
      ot_iovec.h
  11. 19
      ot_livesync.c
  12. 4
      ot_livesync.h
  13. 2
      ot_mutex.c
  14. 13
      ot_mutex.h
  15. 71
      ot_stats.c
  16. 4
      ot_stats.h
  17. 2
      ot_udp.c
  18. 239
      ot_vector.c
  19. 20
      ot_vector.h
  20. 12
      tests/testsuite.sh
  21. 2
      tests/testsuite2.sh
  22. 291
      trackerlogic.c
  23. 32
      trackerlogic.h

7
Makefile

@ -22,16 +22,13 @@ BINDIR?=$(PREFIX)/bin
#FEATURES+=-DWANT_ACCESSLIST_BLACK
#FEATURES+=-DWANT_ACCESSLIST_WHITE
#FEATURES+=-DWANT_SYNC_BATCH
#FEATURES+=-DWANT_SYNC_LIVE
#FEATURES+=-DWANT_UTORRENT1600_WORKAROUND
#FEATURES+=-DWANT_IP_FROM_QUERY_STRING
#FEATURES+=-DWANT_COMPRESSION_GZIP
#FEATURES+=-DWANT_LOG_NETWORKS
#FEATURES+=-DWANT_RESTRICT_STATS
#FEATURES+=-D_DEBUG_HTTPERROR
#FEATURES+=-D_DEBUG_VECTOR
FEATURES+=-DWANT_FULLSCRAPE
@ -42,8 +39,8 @@ CFLAGS+=-I$(LIBOWFAT_HEADERS) -Wall -pipe -Wextra #-pedantic -ansi
LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread -lz
BINARY =opentracker
HEADERS=trackerlogic.h scan_urlencoded_query.h ot_mutex.h ot_stats.h ot_sync.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h
SOURCES=opentracker.c trackerlogic.c scan_urlencoded_query.c ot_mutex.c ot_stats.c ot_sync.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c
HEADERS=trackerlogic.h scan_urlencoded_query.h ot_mutex.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h
SOURCES=opentracker.c trackerlogic.c scan_urlencoded_query.c ot_mutex.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c
OBJECTS = $(SOURCES:%.c=%.o)
OBJECTS_debug = $(SOURCES:%.c=%.debug.o)

42
opentracker.c

@ -5,42 +5,36 @@
$Id$ */
/* System */
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <unistd.h>
#include <stdlib.h>
#include <errno.h>
#include <signal.h>
#include <stdio.h>
#include <pwd.h>
#include <ctype.h>
#include <arpa/inet.h>
/* Libowfat */
#include "socket.h"
#include "io.h"
#include "iob.h"
#include "array.h"
#include "byte.h"
#include "fmt.h"
#include "scan.h"
#include "ip4.h"
/* Opentracker */
#include "trackerlogic.h"
#include "ot_iovec.h"
#include "ot_mutex.h"
#include "ot_http.h"
#include "ot_udp.h"
#include "ot_clean.h"
#include "ot_accesslist.h"
#include "ot_stats.h"
#include "ot_livesync.h"
/* Globals */
time_t g_now;
time_t g_now_seconds;
char * g_redirecturl = NULL;
uint32_t g_tracker_id;
@ -61,7 +55,7 @@ static void signal_handler( int s ) {
trackerlogic_deinit();
exit( 0 );
} else if( s == SIGALRM ) {
g_now = time(NULL);
g_now_seconds = time(NULL);
alarm(5);
}
}
@ -135,7 +129,7 @@ static ssize_t handle_read( const int64 clientsocket ) {
if( array_failed( &h->request ) )
return http_issue_error( clientsocket, CODE_HTTPERROR_500 );
if( ( array_bytes( &h->request ) > 8192 ) && !accesslist_isblessed( (char*)&h->ip, OT_PERMISSION_MAY_SYNC ) )
if( array_bytes( &h->request ) > 8192 )
return http_issue_error( clientsocket, CODE_HTTPERROR_500 );
if( memchr( array_start( &h->request ), '\n', array_bytes( &h->request ) ) )
@ -178,7 +172,7 @@ static void handle_accept( const int64 serversocket ) {
/* That breaks taia encapsulation. But there is no way to take system
time this often in FreeBSD and libowfat does not allow to set unix time */
taia_uint( &t, 0 ); /* Clear t */
tai_unix( &(t.sec), (g_now + OT_CLIENT_TIMEOUT) );
tai_unix( &(t.sec), (g_now_seconds + OT_CLIENT_TIMEOUT) );
io_timeout( i, t );
}
@ -187,8 +181,7 @@ static void handle_accept( const int64 serversocket ) {
}
static void server_mainloop( ) {
static time_t ot_last_clean_time;
time_t next_timeout_check = g_now + OT_CLIENT_TIMEOUT_CHECKINTERVAL;
time_t next_timeout_check = g_now_seconds + OT_CLIENT_TIMEOUT_CHECKINTERVAL;
struct iovec *iovector;
int iovec_entries;
@ -213,20 +206,14 @@ static void server_mainloop( ) {
while( ( i = io_canwrite( ) ) != -1 )
handle_write( i );
if( g_now > next_timeout_check ) {
if( g_now_seconds > next_timeout_check ) {
while( ( i = io_timeouted() ) != -1 )
handle_dead( i );
next_timeout_check = g_now + OT_CLIENT_TIMEOUT_CHECKINTERVAL;
next_timeout_check = g_now_seconds + OT_CLIENT_TIMEOUT_CHECKINTERVAL;
}
livesync_ticker();
/* See if we need to move our pools */
if( NOW != ot_last_clean_time ) {
ot_last_clean_time = NOW;
clean_all_torrents();
}
/* Enforce setting the clock */
signal_handler( SIGALRM );
}
@ -266,7 +253,7 @@ char * set_config_option( char **option, char *value ) {
fprintf( stderr, "Setting config option: %s\n", value );
#endif
while( isspace(*value) ) ++value;
if( *option ) free( *option );
free( *option );
return *option = strdup( value );
}
@ -342,11 +329,6 @@ int parse_configfile( char * config_filename ) {
#endif
} else if(!byte_diff(p, 20, "tracker.redirect_url" ) && isspace(p[20])) {
set_config_option( &g_redirecturl, p+21 );
#ifdef WANT_SYNC_BATCH
} else if(!byte_diff(p, 26, "batchsync.cluster.admin_ip" ) && isspace(p[26])) {
if(!scan_ip4( p+27, tmpip )) goto parse_error;
accesslist_blessip( tmpip, OT_PERMISSION_MAY_SYNC );
#endif
#ifdef WANT_SYNC_LIVE
} else if(!byte_diff(p, 24, "livesync.cluster.node_ip" ) && isspace(p[24])) {
if( !scan_ip4( p+25, tmpip )) goto parse_error;
@ -408,7 +390,7 @@ while( scanon ) {
break;
case 'f': bound += parse_configfile( optarg ); break;
case 'h': help( argv[0] ); exit( 0 );
case 'v': write( 2, static_inbuf, stats_return_tracker_version( static_inbuf )); exit( 0 );
case 'v': stats_return_tracker_version( static_inbuf ); fputs( static_inbuf, stderr ); exit( 0 );
default:
case '?': usage( argv[0] ); exit( 1 );
}
@ -435,7 +417,7 @@ while( scanon ) {
signal( SIGINT, signal_handler );
signal( SIGALRM, signal_handler );
g_now = time( NULL );
g_now_seconds = time( NULL );
if( trackerlogic_init( g_serverdir ? g_serverdir : "." ) == -1 )
panic( "Logic not started" );

11
opentracker.xcodeproj/project.pbxproj

@ -14,7 +14,6 @@
654A808A0CD832FD009035DE /* scan_urlencoded_query.c in Sources */ = {isa = PBXBuildFile; fileRef = 654A80850CD832FC009035DE /* scan_urlencoded_query.c */; };
654A808B0CD832FD009035DE /* trackerlogic.c in Sources */ = {isa = PBXBuildFile; fileRef = 654A80870CD832FC009035DE /* trackerlogic.c */; };
65542D8B0CE078E800469330 /* ot_vector.c in Sources */ = {isa = PBXBuildFile; fileRef = 65542D8A0CE078E800469330 /* ot_vector.c */; };
65542D8E0CE07BA900469330 /* ot_sync.c in Sources */ = {isa = PBXBuildFile; fileRef = 65542D8D0CE07BA900469330 /* ot_sync.c */; };
65542D930CE07CED00469330 /* ot_mutex.c in Sources */ = {isa = PBXBuildFile; fileRef = 65542D8F0CE07CED00469330 /* ot_mutex.c */; };
65542D940CE07CED00469330 /* ot_stats.c in Sources */ = {isa = PBXBuildFile; fileRef = 65542D910CE07CED00469330 /* ot_stats.c */; };
65542E750CE08B9100469330 /* ot_clean.c in Sources */ = {isa = PBXBuildFile; fileRef = 65542E740CE08B9100469330 /* ot_clean.c */; };
@ -53,8 +52,6 @@
654A80880CD832FC009035DE /* trackerlogic.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = trackerlogic.h; sourceTree = "<group>"; };
65542D890CE078E800469330 /* ot_vector.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ot_vector.h; sourceTree = "<group>"; };
65542D8A0CE078E800469330 /* ot_vector.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = ot_vector.c; sourceTree = "<group>"; };
65542D8C0CE07BA900469330 /* ot_sync.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ot_sync.h; sourceTree = "<group>"; };
65542D8D0CE07BA900469330 /* ot_sync.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = ot_sync.c; sourceTree = "<group>"; };
65542D8F0CE07CED00469330 /* ot_mutex.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = ot_mutex.c; sourceTree = "<group>"; };
65542D900CE07CED00469330 /* ot_mutex.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ot_mutex.h; sourceTree = "<group>"; };
65542D910CE07CED00469330 /* ot_stats.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = ot_stats.c; sourceTree = "<group>"; };
@ -114,7 +111,6 @@
653A56B40CE28EC5000CF140 /* ot_iovec.c */,
65542D8F0CE07CED00469330 /* ot_mutex.c */,
65542D910CE07CED00469330 /* ot_stats.c */,
65542D8D0CE07BA900469330 /* ot_sync.c */,
65542EE70CE0CA6B00469330 /* ot_udp.c */,
65542D8A0CE078E800469330 /* ot_vector.c */,
654A80850CD832FC009035DE /* scan_urlencoded_query.c */,
@ -144,9 +140,8 @@
653A56B30CE28EC5000CF140 /* ot_iovec.h */,
65542D900CE07CED00469330 /* ot_mutex.h */,
65542D920CE07CED00469330 /* ot_stats.h */,
65542D8C0CE07BA900469330 /* ot_sync.h */,
65542EE60CE0CA6B00469330 /* ot_udp.h */,
65542D890CE078E800469330 /* ot_vector.h */,
65542EE60CE0CA6B00469330 /* ot_udp.h */,
654A80860CD832FC009035DE /* scan_urlencoded_query.h */,
654A80880CD832FC009035DE /* trackerlogic.h */,
);
@ -244,7 +239,6 @@
654A808A0CD832FD009035DE /* scan_urlencoded_query.c in Sources */,
654A808B0CD832FD009035DE /* trackerlogic.c in Sources */,
65542D8B0CE078E800469330 /* ot_vector.c in Sources */,
65542D8E0CE07BA900469330 /* ot_sync.c in Sources */,
65542D930CE07CED00469330 /* ot_mutex.c in Sources */,
65542D940CE07CED00469330 /* ot_stats.c in Sources */,
65542E750CE08B9100469330 /* ot_clean.c in Sources */,
@ -282,6 +276,7 @@
isa = XCBuildConfiguration;
buildSettings = {
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
GCC_ENABLE_FIX_AND_CONTINUE = YES;
GCC_MODEL_TUNING = G5;
INSTALL_PATH = /usr/local/bin;
LIBRARY_SEARCH_PATHS = (
@ -295,6 +290,7 @@
1DEB928A08733DD80010E9CD /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
GCC_PREPROCESSOR_DEFINITIONS = WANT_IP_FROM_QUERY_STRING;
GCC_WARN_ABOUT_RETURN_TYPE = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
HEADER_SEARCH_PATHS = ../libowfat/;
@ -309,6 +305,7 @@
buildSettings = {
ARCHS = ppc;
DEAD_CODE_STRIPPING = NO;
GCC_PREPROCESSOR_DEFINITIONS = WANT_IP_FROM_QUERY_STRING;
GCC_WARN_ABOUT_RETURN_TYPE = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
HEADER_SEARCH_PATHS = ../libowfat/;

2
ot_accesslist.c

@ -16,6 +16,7 @@
/* Opentracker */
#include "trackerlogic.h"
#include "ot_accesslist.h"
#include "ot_vector.h"
/* GLOBAL VARIABLES */
#ifdef WANT_ACCESSLIST
@ -110,7 +111,6 @@ int accesslist_blessip( char *ip, ot_permissions permissions ) {
uint8_t *_ip = (uint8_t*)ip;
fprintf( stderr, "Blessing ip address %d.%d.%d.%d with:", _ip[0], _ip[1], _ip[2], _ip[3]);
if( permissions & OT_PERMISSION_MAY_STAT ) fputs( " may_fetch_stats", stderr );
if( permissions & OT_PERMISSION_MAY_SYNC ) fputs( " may_sync_batch", stderr );
if( permissions & OT_PERMISSION_MAY_LIVESYNC ) fputs( " may_sync_live", stderr );
if( permissions & OT_PERMISSION_MAY_FULLSCRAPE ) fputs( " may_fetch_fullscrapes", stderr );
if( !permissions ) fputs(" nothing.\n", stderr); else fputs(".\n", stderr );

7
ot_accesslist.h

@ -7,7 +7,7 @@
#define __OT_ACCESSLIST_H__
#if defined ( WANT_ACCESSLIST_BLACK ) && defined (WANT_ACCESSLIST_WHITE )
#error WANT_ACCESSLIST_BLACK and WANT_ACCESSLIST_WHITE are exclusive.
# error WANT_ACCESSLIST_BLACK and WANT_ACCESSLIST_WHITE are exclusive.
#endif
#if defined ( WANT_ACCESSLIST_BLACK ) || defined (WANT_ACCESSLIST_WHITE )
@ -24,9 +24,8 @@ extern char *g_accesslist_filename;
typedef enum {
OT_PERMISSION_MAY_FULLSCRAPE = 0x1,
OT_PERMISSION_MAY_SYNC = 0x2,
OT_PERMISSION_MAY_STAT = 0x4,
OT_PERMISSION_MAY_LIVESYNC = 0x8
OT_PERMISSION_MAY_STAT = 0x2,
OT_PERMISSION_MAY_LIVESYNC = 0x4
} ot_permissions;
int accesslist_blessip( char * ip, ot_permissions permissions );

148
ot_clean.c

@ -7,29 +7,53 @@
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <sys/uio.h>
#include <unistd.h>
#include <stdint.h>
/* Libowfat */
#include "byte.h"
#include "io.h"
/* Opentracker */
#include "trackerlogic.h"
#include "ot_mutex.h"
#include "ot_vector.h"
#include "ot_clean.h"
/* Returns amount of removed peers */
static ssize_t clean_single_bucket( ot_peer *peers, size_t peer_count, time_t timedout, int *removed_seeders ) {
ot_peer *last_peer = peers + peer_count, *insert_point;
time_t timediff;
/* Two scan modes: unless there is one peer removed, just increase ot_peertime */
while( peers < last_peer ) {
if( ( timediff = timedout + OT_PEERTIME( peers ) ) >= OT_PEER_TIMEOUT )
break;
OT_PEERTIME( peers++ ) = timediff;
}
/* If we at least remove one peer, we have to copy */
insert_point = peers;
while( peers < last_peer )
if( ( timediff = timedout + OT_PEERTIME( peers ) ) < OT_PEER_TIMEOUT ) {
OT_PEERTIME( peers ) = timediff;
*(uint64_t*)(insert_point++) = *(uint64_t*)(peers++);
} else
if( OT_FLAG( peers++ ) & PEER_FLAG_SEEDING )
(*removed_seeders)++;
return peers - insert_point;
}
/* Clean a single torrent
return 1 if torrent timed out
*/
int clean_single_torrent( ot_torrent *torrent ) {
ot_peerlist *peer_list = torrent->peer_list;
size_t peers_count = 0, seeds_count;
time_t timedout = (int)( NOW - peer_list->base );
int i;
#ifdef WANT_SYNC_BATCH
char *new_peers;
#endif
ot_vector *bucket_list = &peer_list->peers;
time_t timedout = (time_t)( g_now_minutes - peer_list->base );
int num_buckets = 1, removed_seeders = 0;
/* No need to clean empty torrent */
if( !timedout )
return 0;
@ -38,97 +62,67 @@ int clean_single_torrent( ot_torrent *torrent ) {
return 1;
/* Nothing to be cleaned here? Test if torrent is worth keeping */
if( timedout > OT_POOLS_COUNT ) {
if( timedout > OT_PEER_TIMEOUT ) {
if( !peer_list->peer_count )
return peer_list->down_count ? 0 : 1;
timedout = OT_POOLS_COUNT;
timedout = OT_PEER_TIMEOUT;
}
/* Release vectors that have timed out */
for( i = OT_POOLS_COUNT - timedout; i < OT_POOLS_COUNT; ++i )
free( peer_list->peers[i].data);
/* Shift vectors back by the amount of pools that were shifted out */
memmove( peer_list->peers + timedout, peer_list->peers, sizeof( ot_vector ) * ( OT_POOLS_COUNT - timedout ) );
byte_zero( peer_list->peers, sizeof( ot_vector ) * timedout );
/* Shift back seed counts as well */
memmove( peer_list->seed_counts + timedout, peer_list->seed_counts, sizeof( size_t ) * ( OT_POOLS_COUNT - timedout ) );
byte_zero( peer_list->seed_counts, sizeof( size_t ) * timedout );
#ifdef WANT_SYNC_BATCH
/* Save the block modified within last OT_POOLS_TIMEOUT */
if( peer_list->peers[1].size &&
( new_peers = realloc( peer_list->changeset.data, sizeof( ot_peer ) * peer_list->peers[1].size ) ) )
{
memmove( new_peers, peer_list->peers[1].data, peer_list->peers[1].size );
peer_list->changeset.data = new_peers;
peer_list->changeset.size = sizeof( ot_peer ) * peer_list->peers[1].size;
} else {
free( peer_list->changeset.data );
memset( &peer_list->changeset, 0, sizeof( ot_vector ) );
if( OT_PEERLIST_HASBUCKETS( peer_list ) ) {
num_buckets = bucket_list->size;
bucket_list = (ot_vector *)bucket_list->data;
}
#endif
peers_count = seeds_count = 0;
for( i = 0; i < OT_POOLS_COUNT; ++i ) {
peers_count += peer_list->peers[i].size;
seeds_count += peer_list->seed_counts[i];
while( num_buckets-- ) {
size_t removed_peers = clean_single_bucket( bucket_list->data, bucket_list->size, timedout, &removed_seeders );
peer_list->peer_count -= removed_peers;
bucket_list->size -= removed_peers;
if( bucket_list->size < removed_peers )
vector_fixup_peers( bucket_list );
++bucket_list;
}
peer_list->seed_count = seeds_count;
peer_list->peer_count = peers_count;
if( peers_count )
peer_list->base = NOW;
peer_list->seed_count -= removed_seeders;
/* See, if we need to convert a torrent from simple vector to bucket list */
if( ( peer_list->peer_count > OT_PEER_BUCKET_MINCOUNT ) || OT_PEERLIST_HASBUCKETS(peer_list) )
vector_redistribute_buckets( peer_list );
if( peer_list->peer_count )
peer_list->base = g_now_minutes;
else {
/* When we got here, the last time that torrent
has been touched is OT_POOLS_COUNT units before */
peer_list->base = NOW - OT_POOLS_COUNT;
has been touched is OT_PEER_TIMEOUT Minutes before */
peer_list->base = g_now_minutes - OT_PEER_TIMEOUT;
}
return 0;
}
static void clean_make() {
int bucket;
for( bucket = OT_BUCKET_COUNT - 1; bucket >= 0; --bucket ) {
ot_vector *torrents_list = mutex_bucket_lock( bucket );
size_t toffs;
for( toffs=0; toffs<torrents_list->size; ++toffs ) {
ot_torrent *torrent = ((ot_torrent*)(torrents_list->data)) + toffs;
if( clean_single_torrent( torrent ) ) {
vector_remove_torrent( torrents_list, torrent );
--toffs; continue;
}
}
mutex_bucket_unlock( bucket );
/* We want the cleanup to be spread about 2 Minutes to reduce load spikes
during cleanup. Sleeping around two minutes was chosen to allow enough
time for the actual work and fluctuations in timer. */
usleep( ( 2 * 60 * 1000000 ) / OT_BUCKET_COUNT );
}
}
/* Clean up all peers in current bucket, remove timedout pools and
torrents */
torrents */
static void * clean_worker( void * args ) {
args = args;
args=args;
while( 1 ) {
ot_tasktype tasktype = TASK_CLEAN;
ot_taskid taskid = mutex_workqueue_poptask( &tasktype );
clean_make( );
mutex_workqueue_pushsuccess( taskid );
int bucket = OT_BUCKET_COUNT;
while( bucket-- ) {
ot_vector *torrents_list = mutex_bucket_lock( bucket );
size_t toffs;
for( toffs=0; toffs<torrents_list->size; ++toffs ) {
ot_torrent *torrent = ((ot_torrent*)(torrents_list->data)) + toffs;
if( clean_single_torrent( torrent ) ) {
vector_remove_torrent( torrents_list, torrent );
--toffs; continue;
}
}
mutex_bucket_unlock( bucket );
usleep( OT_CLEAN_SLEEP );
}
}
return NULL;
}
void clean_all_torrents( ) {
mutex_workqueue_pushtask( 0, TASK_CLEAN );
}
static pthread_t thread_id;
void clean_init( void ) {
pthread_create( &thread_id, NULL, clean_worker, NULL );

8
ot_clean.h

@ -6,10 +6,14 @@
#ifndef __OT_CLEAN_H__
#define __OT_CLEAN_H__
/* The amount of time a clean cycle should take */
#define OT_CLEAN_INTERVAL_MINUTES 2
/* So after each bucket wait 1 / OT_BUCKET_COUNT intervals */
#define OT_CLEAN_SLEEP ( ( ( OT_CLEAN_INTERVAL_MINUTES ) * 60 * 1000000 ) / ( OT_BUCKET_COUNT ) )
void clean_init( void );
void clean_deinit( void );
void clean_all_torrents( void );
int clean_single_torrent( ot_torrent *torrent );
#endif

1
ot_fullscrape.c

@ -7,7 +7,6 @@
/* System */
#include <sys/param.h>
#include <sys/uio.h>
#include <stdio.h>
#include <string.h>
#include <pthread.h>

62
ot_http.c

@ -5,7 +5,6 @@
/* System */
#include <sys/types.h>
#include <sys/uio.h>
#include <arpa/inet.h>
#include <stdlib.h>
#include <stdio.h>
@ -26,7 +25,6 @@
#include "ot_fullscrape.h"
#include "ot_stats.h"
#include "ot_accesslist.h"
#include "ot_sync.h"
#define OT_MAXMULTISCRAPE_COUNT 64
static ot_hash multiscrape_buf[OT_MAXMULTISCRAPE_COUNT];
@ -165,52 +163,6 @@ ssize_t http_sendiovecdata( const int64 client_socket, int iovec_entries, struct
return 0;
}
#ifdef WANT_SYNC_BATCH
static ssize_t http_handle_sync( const int64 client_socket, char *data ) {
struct http_data* h = io_getcookie( client_socket );
size_t len;
int mode = SYNC_OUT, scanon = 1;
char *c = data;
if( !accesslist_isblessed( h->ip, OT_PERMISSION_MAY_SYNC ) )
HTTPERROR_403_IP;
while( scanon ) {
switch( scan_urlencoded_query( &c, data = c, SCAN_SEARCHPATH_PARAM ) ) {
case -2: scanon = 0; break; /* TERMINATOR */
case -1: HTTPERROR_400_PARAM; /* PARSE ERROR */
default: scan_urlencoded_skipvalue( &c ); break;
case 9:
if(byte_diff(data,9,"changeset")) {
scan_urlencoded_skipvalue( &c );
continue;
}
/* ignore this, when we dont at least see "d4:syncdee" */
if( ( len = scan_urlencoded_query( &c, data = c, SCAN_SEARCHPATH_VALUE ) ) < 10 ) HTTPERROR_400_PARAM;
if( add_changeset_to_tracker( (uint8_t*)data, len ) ) HTTPERROR_400_PARAM;
if( mode == SYNC_OUT ) {
stats_issue_event( EVENT_SYNC_IN, FLAG_TCP, 0 );
mode = SYNC_IN;
}
break;
}
}
if( mode == SYNC_OUT ) {
/* Pass this task to the worker thread */
h->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK;
stats_issue_event( EVENT_SYNC_OUT_REQUEST, FLAG_TCP, 0 );
sync_deliver( client_socket );
io_dontwantread( client_socket );
return -2;
}
/* Simple but proof for now */
memmove( static_outbuf + SUCCESS_HTTP_HEADER_LENGTH, "OK", 2);
return 2;
}
#endif
static ssize_t http_handle_stats( const int64 client_socket, char *data, char *d, size_t l ) {
char *c = data;
int mode = TASK_STATS_PEERS, scanon = 1, format = 0;
@ -245,10 +197,6 @@ static ssize_t http_handle_stats( const int64 client_socket, char *data, char *d
mode = TASK_STATS_UDP;
else if( !byte_diff(data,4,"busy"))
mode = TASK_STATS_BUSY_NETWORKS;
else if( !byte_diff(data,4,"dmem"))
mode = TASK_STATS_MEMORY;
else if( !byte_diff(data,4,"vdeb"))
mode = TASK_STATS_VECTOR_DEBUG;
else if( !byte_diff(data,4,"torr"))
mode = TASK_STATS_TORRENTS;
else if( !byte_diff(data,4,"fscr"))
@ -265,7 +213,7 @@ static ssize_t http_handle_stats( const int64 client_socket, char *data, char *d
case 5:
if( !byte_diff(data,5,"top10"))
mode = TASK_STATS_TOP10;
if( !byte_diff(data,5,"renew"))
else if( !byte_diff(data,5,"renew"))
mode = TASK_STATS_RENEW;
else
HTTPERROR_400_PARAM;
@ -524,7 +472,7 @@ static ssize_t http_handle_announce( const int64 client_socket, char *data ) {
len = remove_peer_from_torrent( hash, &peer, SUCCESS_HTTP_HEADER_LENGTH + static_outbuf, FLAG_TCP );
else {
torrent = add_peer_to_torrent( hash, &peer WANT_SYNC_PARAM( 0 ) );
if( !torrent || !( len = return_peers_for_torrent( hash, numwant, SUCCESS_HTTP_HEADER_LENGTH + static_outbuf, FLAG_TCP ) ) ) HTTPERROR_500;
if( !torrent || !( len = return_peers_for_torrent( torrent, numwant, SUCCESS_HTTP_HEADER_LENGTH + static_outbuf, FLAG_TCP ) ) ) HTTPERROR_500;
}
stats_issue_event( EVENT_ANNOUNCE, FLAG_TCP, len);
return len;
@ -573,12 +521,6 @@ ssize_t http_handle_request( const int64 client_socket, char *data, size_t recv_
reply_size = http_handle_scrape( client_socket, c );
/* All the rest is matched the standard way */
else switch( len ) {
#ifdef WANT_SYNC_BATCH
case 4: /* sync ? */
if( byte_diff( data, 4, "sync") ) HTTPERROR_404;
reply_size = http_handle_sync( client_socket, c );
break;
#endif
case 5: /* stats ? */
if( byte_diff( data, 5, "stats") ) HTTPERROR_404;
reply_size = http_handle_stats( client_socket, c, recv_header, recv_length );

2
ot_iovec.h

@ -6,6 +6,8 @@
#ifndef __OT_IOVEC_H__
#define __OT_IOVEC_H__
#include <sys/uio.h>
void *iovec_increase( int *iovec_entries, struct iovec **iovector, size_t new_alloc );
void iovec_fixlast( int *iovec_entries, struct iovec **iovector, void *last_ptr );
void iovec_free( int *iovec_entries, struct iovec **iovector );

19
ot_livesync.c

@ -50,7 +50,7 @@ void livesync_init( ) {
livesync_outbuffer_pos = livesync_outbuffer_start;
memmove( livesync_outbuffer_pos, &g_tracker_id, sizeof( g_tracker_id ) );
livesync_outbuffer_pos += sizeof( g_tracker_id );
livesync_lastpacket_time = g_now;
livesync_lastpacket_time = g_now_seconds;
pthread_create( &thread_id, NULL, livesync_worker, NULL );
}
@ -88,14 +88,13 @@ static void livesync_issuepacket( ) {
socket_send4(g_livesync_socket_out, (char*)livesync_outbuffer_start, livesync_outbuffer_pos - livesync_outbuffer_start,
groupip_1, LIVESYNC_PORT);
livesync_outbuffer_pos = livesync_outbuffer_start + sizeof( g_tracker_id );
livesync_lastpacket_time = g_now;
livesync_lastpacket_time = g_now_seconds;
}
/* Inform live sync about whats going on. */
void livesync_tell( ot_hash * const info_hash, const ot_peer * const peer, const uint8_t peerflag ) {
void livesync_tell( ot_hash * const info_hash, const ot_peer * const peer ) {
memmove( livesync_outbuffer_pos , info_hash, sizeof(ot_hash));
memmove( livesync_outbuffer_pos + sizeof(ot_hash), peer, sizeof(ot_peer));
OT_FLAG( livesync_outbuffer_pos + sizeof(ot_hash) ) |= peerflag;
livesync_outbuffer_pos += sizeof(ot_hash) + sizeof(ot_peer);
if( livesync_outbuffer_pos >= livesync_outbuffer_highwater )
@ -106,7 +105,7 @@ void livesync_tell( ot_hash * const info_hash, const ot_peer * const peer, const
stuck when there's not enough traffic to fill udp packets fast
enough */
void livesync_ticker( ) {
if( ( g_now - livesync_lastpacket_time > LIVESYNC_MAXDELAY) &&
if( ( g_now_seconds - livesync_lastpacket_time > LIVESYNC_MAXDELAY) &&
( livesync_outbuffer_pos > livesync_outbuffer_start + sizeof( g_tracker_id ) ) )
livesync_issuepacket();
}
@ -126,22 +125,22 @@ static void * livesync_worker( void * args ) {
continue;
if( datalen < (ssize_t)(sizeof( g_tracker_id ) + sizeof( ot_hash ) + sizeof( ot_peer ) ) ) {
// TODO: log invalid sync packet
/* TODO: log invalid sync packet */
continue;
}
if( !accesslist_isblessed((char*)in_ip, OT_PERMISSION_MAY_LIVESYNC)) {
// TODO: log invalid sync packet
/* TODO: log invalid sync packet */
continue;
}
if( !memcmp( livesync_inbuffer, &g_tracker_id, sizeof( g_tracker_id ) ) ) {
// TODO: log packet coming from ourselves
/* TODO: log packet coming from ourselves */
continue;
}
// Now basic sanity checks have been done on the live sync packet
// We might add more testing and logging.
/* Now basic sanity checks have been done on the live sync packet
We might add more testing and logging. */
while( off + (ssize_t)sizeof( ot_hash ) + (ssize_t)sizeof( ot_peer ) <= datalen ) {
ot_peer *peer = (ot_peer*)(livesync_inbuffer + off + sizeof(ot_hash));
ot_hash *hash = (ot_hash*)(livesync_inbuffer + off);

4
ot_livesync.h

@ -35,7 +35,6 @@
]+
]*
*/
#ifdef WANT_SYNC_LIVE
@ -49,7 +48,7 @@ void livesync_deinit();
void livesync_bind_mcast( char *ip, uint16_t port );
/* Inform live sync about whats going on. */
void livesync_tell( ot_hash * const info_hash, const ot_peer * const peer, const uint8_t peerflag );
void livesync_tell( ot_hash * const info_hash, const ot_peer * const peer );
/* Tickle the live sync module from time to time, so no events get
stuck when there's not enough traffic to fill udp packets fast
@ -63,7 +62,6 @@ void handle_livesync( const int64 serversocket );
/* If no syncing is required, save calling code from #ifdef
constructions */
#define livesync_init()
#define livesync_ticker()
#define handle_livesync(a)

2
ot_mutex.c

@ -174,7 +174,7 @@ void mutex_workqueue_canceltask( int64 socket ) {
/* Free task's iovec */
for( i=0; i<(*task)->iovec_entries; ++i )
munmap( iovec[i].iov_base , iovec[i].iov_len );
munmap( iovec[i].iov_base, iovec[i].iov_len );
*task = (*task)->next;
free( ptask );

13
ot_mutex.h

@ -6,6 +6,8 @@
#ifndef __OT_MUTEX_H__
#define __OT_MUTEX_H__
#include <sys/uio.h>
void mutex_init( );
void mutex_deinit( );
@ -27,27 +29,20 @@ typedef enum {
TASK_STATS_TORADDREM = 0x0009,
TASK_STATS_VERSION = 0x000a,
TASK_STATS_BUSY_NETWORKS = 0x000b,
TASK_STATS_VECTOR_DEBUG = 0x000c,
TASK_STATS_RENEW = 0x000d,
TASK_STATS_RENEW = 0x000c,
TASK_STATS = 0x0100, /* Mask */
TASK_STATS_TORRENTS = 0x0101,
TASK_STATS_PEERS = 0x0102,
TASK_STATS_SLASH24S = 0x0103,
TASK_STATS_TOP10 = 0x0104,
TASK_STATS_MEMORY = 0x0105,
TASK_FULLSCRAPE = 0x0200, /* Default mode */
TASK_FULLSCRAPE_TPB_BINARY = 0x0201,
TASK_FULLSCRAPE_TPB_ASCII = 0x0202,
TASK_FULLSCRAPE_TPB_URLENCODED = 0x0203,
TASK_CLEAN = 0x0300,
TASK_SYNC_OUT = 0x0400,
TASK_SYNC_IN = 0x0401,
TASK_DMEM = 0x0500,
TASK_DMEM = 0x0300,
TASK_DONE = 0x0f00,

71
ot_stats.c

@ -46,7 +46,7 @@ static unsigned long long ot_full_scrape_count = 0;
static unsigned long long ot_full_scrape_request_count = 0;
static unsigned long long ot_full_scrape_size = 0;
static unsigned long long ot_failed_request_counts[CODE_HTTPERROR_COUNT];
static unsigned long long ot_renewed[OT_POOLS_COUNT];
static unsigned long long ot_renewed[OT_PEER_TIMEOUT];
static time_t ot_start_time;
@ -214,7 +214,7 @@ static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh )
uint32_t *counts[ NUM_BUFS ];
uint32_t slash24s[amount*2]; /* first dword amount, second dword subnet */
int bucket;
// int bucket;
size_t i, j, k, l;
char *r = reply;
@ -223,6 +223,8 @@ static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh )
r += sprintf( r, "Stats for all /24s with more than %u announced torrents:\n\n", thresh );
#if 0
/* XXX: TOOD: Doesn't work yet with new peer storage model */
for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
ot_vector *torrents_list = mutex_bucket_lock( bucket );
for( j=0; j<torrents_list->size; ++j ) {
@ -248,6 +250,7 @@ static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh )
}
mutex_bucket_unlock( bucket );
}
#endif
k = l = 0; /* Debug: count allocated bufs */
for( i=0; i < NUM_BUFS; ++i ) {
@ -283,8 +286,6 @@ static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh )
return r - reply;
bailout_cleanup:
for( i=0; i < NUM_BUFS; ++i )
free( counts[i] );
@ -299,44 +300,6 @@ bailout_cleanup:
}
*/
static ssize_t stats_vector_usage( char * reply ) {
size_t i, j, *vec_member;
char *r = reply;
int exactmatch, bucket;
ot_vector bucketsizes;
memset( &bucketsizes, 0, sizeof( bucketsizes ));
for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
ot_vector *torrents_list = mutex_bucket_lock( bucket );
for( i=0; i<torrents_list->size; ++i ) {
ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[i] ).peer_list;
for( j=0; j<OT_POOLS_COUNT; ++j ) {
if( ! ( vec_member = vector_find_or_insert(&bucketsizes, &peer_list->peers[j].size, 3 * sizeof( size_t ), 2 * sizeof(size_t), &exactmatch) ) ) {
mutex_bucket_unlock( bucket );
return 0;
}
if( !exactmatch ) {
vec_member[0] = peer_list->peers[j].size;
vec_member[1] = peer_list->peers[j].space;
vec_member[2] = 1;
} else
++vec_member[2];
}
}
mutex_bucket_unlock( bucket );
}
for( i = 0; i<bucketsizes.size; ++i ) {
r += sprintf( r, "%zd\t%zd\t%zd\n", ((size_t*)bucketsizes.data)[3*i], ((size_t*)bucketsizes.data)[3*i+1], ((size_t*)bucketsizes.data)[3*i+2] );
/* Prevent overflow. 8k should be enough for debugging */
if( r - reply > OT_STATS_TMPSIZE - 3*10+3 /* 3*%zd + 2*\t + \n */ )
break;
}
return r - reply;
}
static unsigned long events_per_time( unsigned long long events, time_t t ) {
return events / ( (unsigned int)t ? (unsigned int)t : 1 );
}
@ -497,20 +460,20 @@ static size_t stats_return_renew_bucket( char * reply ) {
char *r = reply;
int i;
for( i=0; i<OT_POOLS_COUNT; ++i )
for( i=0; i<OT_PEER_TIMEOUT; ++i )
r+=sprintf(r,"%02i %llu\n", i, ot_renewed[i] );
return r - reply;
}
extern const char
*g_version_opentracker_c, *g_version_accesslist_c, *g_version_clean_c, *g_version_fullscrape_c, *g_version_http_c,
*g_version_iovec_c, *g_version_mutex_c, *g_version_stats_c, *g_version_sync_c, *g_version_udp_c, *g_version_vector_c,
*g_version_iovec_c, *g_version_mutex_c, *g_version_stats_c, *g_version_udp_c, *g_version_vector_c,
*g_version_scan_urlencoded_query_c, *g_version_trackerlogic_c, *g_version_livesync_c;
size_t stats_return_tracker_version( char *reply ) {
return sprintf( reply, "%s%s%s%s%s%s%s%s%s%s%s%s%s%s",
return sprintf( reply, "%s%s%s%s%s%s%s%s%s%s%s%s%s",
g_version_opentracker_c, g_version_accesslist_c, g_version_clean_c, g_version_fullscrape_c, g_version_http_c,
g_version_iovec_c, g_version_mutex_c, g_version_stats_c, g_version_sync_c, g_version_udp_c, g_version_vector_c,
g_version_iovec_c, g_version_mutex_c, g_version_stats_c, g_version_udp_c, g_version_vector_c,
g_version_scan_urlencoded_query_c, g_version_trackerlogic_c, g_version_livesync_c );
}
@ -540,10 +503,6 @@ size_t return_stats_for_tracker( char *reply, int mode, int format ) {
#ifdef WANT_LOG_NETWORKS
case TASK_STATS_BUSY_NETWORKS:
return stats_return_busy_networks( reply );
#endif
#ifdef _DEBUG_VECTOR
case TASK_STATS_VECTOR_DEBUG:
return vector_info( reply );
#endif
default:
return 0;
@ -563,7 +522,6 @@ static void stats_make( int *iovec_entries, struct iovec **iovector, ot_tasktype
case TASK_STATS_PEERS: r += stats_peers_mrtg( r ); break;
case TASK_STATS_SLASH24S: r += stats_slash24s_txt( r, 25, 16 ); break;
case TASK_STATS_TOP10: r += stats_top10_txt( r ); break;
case TASK_STATS_MEMORY: r += stats_vector_usage( r ); break;
default:
iovec_free(iovec_entries, iovector);
return;
@ -594,14 +552,14 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uint32_t event_
case EVENT_FULLSCRAPE_REQUEST:
{
uint8_t ip[4]; *(uint32_t*)ip = (uint32_t)proto; /* ugly hack to transfer ip to stats */
LOG_TO_STDERR( "[%08d] scrp: %d.%d.%d.%d - FULL SCRAPE\n", (unsigned int)(g_now - ot_start_time), ip[0], ip[1], ip[2], ip[3] );
LOG_TO_STDERR( "[%08d] scrp: %d.%d.%d.%d - FULL SCRAPE\n", (unsigned int)(g_now_seconds - ot_start_time)/60, ip[0], ip[1], ip[2], ip[3] );
ot_full_scrape_request_count++;
}
break;
case EVENT_FULLSCRAPE_REQUEST_GZIP:
{
uint8_t ip[4]; *(uint32_t*)ip = (uint32_t)proto; /* ugly hack to transfer ip to stats */
LOG_TO_STDERR( "[%08d] scrp: %d.%d.%d.%d - FULL SCRAPE GZIP\n", (unsigned int)(g_now - ot_start_time), ip[0], ip[1], ip[2], ip[3] );
LOG_TO_STDERR( "[%08d] scrp: %d.%d.%d.%d - FULL SCRAPE GZIP\n", (unsigned int)(g_now_seconds - ot_start_time)/60, ip[0], ip[1], ip[2], ip[3] );
ot_full_scrape_request_count++;
}
break;
@ -611,11 +569,6 @@ void stats_issue_event( ot_status_event event, PROTO_FLAG proto, uint32_t event_
case EVENT_RENEW:
ot_renewed[event_data]++;
break;
case EVENT_SYNC_IN_REQUEST:
case EVENT_SYNC_IN:
case EVENT_SYNC_OUT_REQUEST:
case EVENT_SYNC_OUT:
break;
default:
break;
}
@ -643,7 +596,7 @@ void stats_deliver( int64 socket, int tasktype ) {
static pthread_t thread_id;
void stats_init( ) {
ot_start_time = g_now;
ot_start_time = g_now_seconds;
pthread_create( &thread_id, NULL, stats_worker, NULL );
}

4
ot_stats.h

@ -16,10 +16,6 @@ typedef enum {
EVENT_FULLSCRAPE_REQUEST,
EVENT_FULLSCRAPE_REQUEST_GZIP,
EVENT_FULLSCRAPE, /* TCP only */
EVENT_SYNC_IN_REQUEST,
EVENT_SYNC_IN,
EVENT_SYNC_OUT_REQUEST,
EVENT_SYNC_OUT,
EVENT_FAILED
} ot_status_event;

2
ot_udp.c

@ -115,7 +115,7 @@ void handle_udp4( int64 serversocket ) {
if( !torrent )
return; /* XXX maybe send error */
r = 8 + return_peers_for_torrent( hash, numwant, static_outbuf + 8, FLAG_UDP );
r = 8 + return_peers_for_torrent( torrent, numwant, static_outbuf + 8, FLAG_UDP );
}
socket_send4( serversocket, static_outbuf, r, remoteip, remoteport );

239
ot_vector.c

@ -6,69 +6,65 @@
/* System */
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <stdio.h>
/* Opentracker */
#include "trackerlogic.h"
#include "ot_vector.h"
#ifdef _DEBUG_VECTOR
#include <stdio.h>
static uint64_t vector_debug_inc[32];
static uint64_t vector_debug_noinc[32];
static uint64_t vector_debug_dec[32];
static uint64_t vector_debug_nodec[32];
static void vector_debug( size_t old_size, ssize_t diff_size, size_t old_space, ssize_t diff_space ) {
int x = 0;
while( old_space ) { old_space>>=1; ++x; }
old_size = old_size;
if( diff_size == -1 )
if( diff_space ) vector_debug_dec[x]++; else vector_debug_nodec[x]++;
else
if( diff_space ) vector_debug_inc[x]++; else vector_debug_noinc[x]++;
}
/* Libowfat */
#include "uint32.h"
size_t vector_info( char * reply ) {
char * r = reply;
int i;
for( i=1; i<28; ++i )
r += sprintf( r, " inc % 12d -> % 12d: % 16lld\n", 1<<(i-1), 8<<(i-1), vector_debug_inc[i] );
for( i=1; i<28; ++i )
r += sprintf( r, "noinc % 12d -> % 12d: % 16lld\n", 1<<(i-1), 1<<(i-1), vector_debug_noinc[i] );
for( i=1; i<28; ++i )
r += sprintf( r, " dec % 12d -> % 12d: % 16lld\n", 1<<(i-1), 4<<(i-1), vector_debug_dec[i] );
for( i=1; i<28; ++i )
r += sprintf( r, "nodec % 12d -> % 12d: % 16lld\n", 1<<(i-1), 1<<(i-1), vector_debug_nodec[i] );
return r - reply;
static int vector_compare_peer(const void *peer1, const void *peer2 ) {
int32_t cmp = (int32_t)uint32_read(peer1) - (int32_t)uint32_read(peer2);
if (cmp == 0) cmp = ((int8_t*)peer1)[4] - ((int8_t*)peer2)[4];
if (cmp == 0) cmp = ((int8_t*)peer1)[5] - ((int8_t*)peer2)[5];
return cmp;
}
#endif
/* This function gives us a binary search that returns a pointer, even if
no exact match is found. In that case it sets exactmatch 0 and gives
calling functions the chance to insert data
NOTE: Minimal compare_size is 4.
*/
void *binary_search( const void * const key, const void * base, const size_t member_count, const size_t member_size,
size_t compare_size, int *exactmatch ) {
size_t mc = member_count;
uint8_t *lookat = ((uint8_t*)base) + member_size * (member_count >> 1);
size_t offs, mc = member_count;
int8_t *lookat = ((int8_t*)base) + member_size * (mc >> 1);
int32_t key_cache = (int32_t)uint32_read(key);
*exactmatch = 1;
while( mc ) {
int cmp = memcmp( lookat, key, compare_size);
if (cmp == 0) return (void *)lookat;
int32_t cmp = key_cache - (int32_t)uint32_read(lookat);
if (cmp == 0) {
for( offs = 4; cmp == 0 && offs < compare_size; ++offs )
cmp = ((int8_t*)key)[offs] - lookat[offs];
if( cmp == 0 )
return (void *)lookat;
}
if (cmp < 0) {
base = (void*)(lookat + member_size);
--mc;
}
mc >>= 1;
lookat = ((uint8_t*)base) + member_size * (mc >> 1);
lookat = ((int8_t*)base) + member_size * (mc >> 1);
}
*exactmatch = 0;
return (void*)lookat;
}
static uint8_t vector_hash_peer( ot_peer *peer, int bucket_count ) {
unsigned int hash = 5381, i = 6;
uint8_t *p = (uint8_t*)peer;
while( i-- ) hash += (hash<<5) + *(p++);
return hash % bucket_count;
}
/* This is the generic insert operation for our vector type.
It tries to locate the object at "key" with size "member_size" by comparing its first "compare_size" bytes with
those of objects in vector. Our special "binary_search" function does that and either returns the match or a
@ -78,17 +74,13 @@ void *binary_search( const void * const key, const void * base, const size_t mem
*/
void *vector_find_or_insert( ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch ) {
uint8_t *match = binary_search( key, vector->data, vector->size, member_size, compare_size, exactmatch );
#ifdef _DEBUG_VECTOR
size_t old_space = vector->space;
#endif
if( *exactmatch ) return match;
if( vector->size + 1 >= vector->space ) {
if( vector->size + 1 > vector->space ) {
size_t new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS;
uint8_t *new_data = realloc( vector->data, new_space * member_size );
if( !new_data ) return NULL;
/* Adjust pointer if it moved by realloc */
match = new_data + (match - (uint8_t*)vector->data);
@ -97,56 +89,48 @@ void *vector_find_or_insert( ot_vector *vector, void *key, size_t member_size, s
}
memmove( match + member_size, match, ((uint8_t*)vector->data) + member_size * vector->size - match );
#ifdef _DEBUG_VECTOR
vector_debug( vector->size, 1, old_space, vector->space - old_space );
#endif
vector->size++;
return match;
}
/* This function checks, whether our peer vector is a real vector
or a list of buckets and dispatches accordingly */
ot_peer *vector_find_or_insert_peer( ot_vector *vector, ot_peer *peer, int *exactmatch ) {
/* If space is zero but size is set, we're dealing with a list of vector->size buckets */
if( vector->space < vector->size )
vector = ((ot_vector*)vector->data) + vector_hash_peer(peer, vector->size );
return vector_find_or_insert( vector, peer, sizeof(ot_peer), OT_PEER_COMPARE_SIZE, exactmatch );
}
/* This is the non-generic delete from vector-operation specialized for peers in pools.
Set hysteresis == 0 if you expect the vector not to ever grow again.
It returns 0 if no peer was found (and thus not removed)
1 if a non-seeding peer was removed
2 if a seeding peer was removed
*/
int vector_remove_peer( ot_vector *vector, ot_peer *peer, int hysteresis ) {
int vector_remove_peer( ot_vector *vector, ot_peer *peer ) {
int exactmatch;
size_t shrink_thresh = hysteresis ? OT_VECTOR_SHRINK_THRESH : OT_VECTOR_SHRINK_RATIO;
ot_peer *end = ((ot_peer*)vector->data) + vector->size;
ot_peer *match;
#ifdef _DEBUG_VECTOR
size_t old_space = vector->space;
#endif
ot_peer *match, *end;
if( !vector->size ) return 0;
match = binary_search( peer, vector->data, vector->size, sizeof( ot_peer ), OT_PEER_COMPARE_SIZE, &exactmatch );
/* If space is zero but size is set, we're dealing with a list of vector->size buckets */
if( vector->space < vector->size )
vector = ((ot_vector*)vector->data) + vector_hash_peer(peer, vector->size );
end = ((ot_peer*)vector->data) + vector->size;
match = binary_search( peer, vector->data, vector->size, sizeof( ot_peer ), OT_PEER_COMPARE_SIZE, &exactmatch );
if( !exactmatch ) return 0;
exactmatch = ( OT_FLAG( match ) & PEER_FLAG_SEEDING ) ? 2 : 1;
memmove( match, match + 1, sizeof(ot_peer) * ( end - match - 1 ) );
if( ( --vector->size * shrink_thresh < vector->space ) && ( vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS ) ) {
vector->space /= OT_VECTOR_SHRINK_RATIO;
vector->data = realloc( vector->data, vector->space * sizeof( ot_peer ) );
}
if( !vector->size ) {
/* for peer pools its safe to let them go,
in 999 of 1000 this happens in older pools, that won't ever grow again */
free( vector->data );
vector->data = NULL;
vector->space = 0;
}
#ifdef _DEBUG_VECTOR
vector_debug( vector->size+1, -1, old_space, vector->space - old_space );
#endif
vector->size--;
vector_fixup_peers( vector );
return exactmatch;
}
void vector_remove_torrent( ot_vector *vector, ot_torrent *match ) {
ot_torrent *end = ((ot_torrent*)vector->data) + vector->size;
#ifdef _DEBUG_VECTOR
size_t old_space = vector->space;
#endif
if( !vector->size ) return;
@ -159,9 +143,118 @@ void vector_remove_torrent( ot_vector *vector, ot_torrent *match ) {
vector->space /= OT_VECTOR_SHRINK_RATIO;
vector->data = realloc( vector->data, vector->space * sizeof( ot_torrent ) );
}
#ifdef _DEBUG_VECTOR
vector_debug( vector->size+1, -1, old_space, vector->space - old_space );
#endif
}
void vector_clean_list( ot_vector * vector, int num_buckets ) {
while( num_buckets-- )
free( vector[num_buckets].data );
free( vector );
return;
}
void vector_redistribute_buckets( ot_peerlist * peer_list ) {
int tmp, bucket, bucket_size_new, num_buckets_new, num_buckets_old = 1;
ot_vector * bucket_list_new, * bucket_list_old = &peer_list->peers;
if( OT_PEERLIST_HASBUCKETS( peer_list ) ) {
num_buckets_old = peer_list->peers.size;
bucket_list_old = peer_list->peers.data;
}
if( peer_list->peer_count < 255 )
num_buckets_new = 1;
else if( peer_list->peer_count > 8192 )
num_buckets_new = 64;
else if( peer_list->peer_count >= 512 && peer_list->peer_count < 4096 )
num_buckets_new = 16;
else if( peer_list->peer_count < 512 && num_buckets_old <= 16 )
num_buckets_new = num_buckets_old;
else if( peer_list->peer_count < 512 )
num_buckets_new = 1;
else if( peer_list->peer_count < 8192 && num_buckets_old > 1 )
num_buckets_new = num_buckets_old;
else
num_buckets_new = 16;
if( num_buckets_new == num_buckets_old )
return;
/* Assume near perfect distribution */
bucket_list_new = malloc( num_buckets_new * sizeof( ot_vector ) );
if( !bucket_list_new) return;
bzero( bucket_list_new, num_buckets_new * sizeof( ot_vector ) );
tmp = peer_list->peer_count / num_buckets_new;
bucket_size_new = OT_VECTOR_MIN_MEMBERS;
while( bucket_size_new < tmp)
bucket_size_new *= OT_VECTOR_GROW_RATIO;
/* preallocate vectors to hold all peers */
for( bucket=0; bucket<num_buckets_new; ++bucket ) {
bucket_list_new[bucket].space = bucket_size_new;
bucket_list_new[bucket].data = malloc( bucket_size_new * sizeof(ot_peer) );
if( !bucket_list_new[bucket].data )
return vector_clean_list( bucket_list_new, num_buckets_new );
}
/* Now sort them into the correct bucket */
for( bucket=0; bucket<num_buckets_old; ++bucket ) {
ot_peer * peers_old = bucket_list_old[bucket].data, * peers_new;
int peer_count_old = bucket_list_old[bucket].size;
while( peer_count_old-- ) {
ot_vector * bucket_dest = bucket_list_new;
if( num_buckets_new > 1 )
bucket_dest += vector_hash_peer(peers_old, num_buckets_new);
if( bucket_dest->size + 1 > bucket_dest->space ) {
void * tmp = realloc( bucket_dest->data, sizeof(ot_peer) * OT_VECTOR_GROW_RATIO * bucket_dest->space );
if( !tmp ) return vector_clean_list( bucket_list_new, num_buckets_new );
bucket_dest->data = tmp;
bucket_dest->space *= OT_VECTOR_GROW_RATIO;
}
peers_new = (ot_peer*)bucket_dest->data;
*(uint64_t*)(peers_new + bucket_dest->size++) = *(uint64_t*)(peers_old++);
}
}
/* Now sort each bucket to later allow bsearch */
for( bucket=0; bucket<num_buckets_new; ++bucket )
qsort( bucket_list_new[bucket].data, bucket_list_new[bucket].size, sizeof( ot_peer ), vector_compare_peer );
/* Everything worked fine. Now link new bucket_list to peer_list */
if( OT_PEERLIST_HASBUCKETS( peer_list) )
vector_clean_list( (ot_vector*)peer_list->peers.data, peer_list->peers.size );
else
free( peer_list->peers.data );
if( num_buckets_new > 1 ) {
peer_list->peers.data = bucket_list_new;
peer_list->peers.size = num_buckets_new;
peer_list->peers.space = 0; /* Magic marker for "is list of buckets" */
} else {
peer_list->peers.data = bucket_list_new->data;
peer_list->peers.size = bucket_list_new->size;
peer_list->peers.space = bucket_list_new->space;
free( bucket_list_new );
}
}
void vector_fixup_peers( ot_vector * vector ) {
int need_fix = 0;
if( !vector->size ) {
free( vector->data );
vector->data = NULL;
vector->space = 0;
return;
}
while( ( vector->size * OT_VECTOR_SHRINK_THRESH < vector->space ) &&
( vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS ) ) {
vector->space /= OT_VECTOR_SHRINK_RATIO;
need_fix++;
}
if( need_fix )
vector->data = realloc( vector->data, vector->space * sizeof( ot_peer ) );
}
const char *g_version_vector_c = "$Source$: $Revision$\n";

20
ot_vector.h

@ -12,21 +12,23 @@
#define OT_VECTOR_SHRINK_THRESH 4
#define OT_VECTOR_SHRINK_RATIO 2
#define OT_PEER_BUCKET_MINCOUNT 512
#define OT_PEER_BUCKET_MAXCOUNT 256
typedef struct {
void *data;
size_t size;
size_t space;
} ot_vector;
void *binary_search( const void * const key, const void * base, const size_t member_count, const size_t member_size,
size_t compare_size, int *exactmatch );
void *vector_find_or_insert( ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch );
int vector_remove_peer( ot_vector *vector, ot_peer *peer, int hysteresis );
void vector_remove_torrent( ot_vector *vector, ot_torrent *match );
void *binary_search( const void * const key, const void * base, const size_t member_count, const size_t member_size,
size_t compare_size, int *exactmatch );
void *vector_find_or_insert( ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch );
ot_peer *vector_find_or_insert_peer( ot_vector *vector, ot_peer *peer, int *exactmatch );
#ifdef _DEBUG_VECTOR
size_t vector_info( char * reply );
#endif
int vector_remove_peer( ot_vector *vector, ot_peer *peer );
void vector_remove_torrent( ot_vector *vector, ot_torrent *match );
void vector_redistribute_buckets( ot_peerlist * peer_list );
void vector_fixup_peers( ot_vector * vector );
#endif

12
tests/testsuite.sh

@ -1,15 +1,11 @@
#!/bin/sh
while true; do
request_string="GET /announce?info_hash=\
%$(printf %02X $(( $RANDOM & 0xff )) )\