gecko-dev/webtools/web-sniffer/robot.c
erik%vanderpoel.org 6a5734b52e added support for XHTML's empty elements <foo />;
added support for declarations like <!DOCTYPE ...>;
added support for processing instructions like <?xml ...>
2005-02-03 01:06:24 +00:00

1166 lines
20 KiB
C

/* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is SniffURI.
*
* The Initial Developer of the Original Code is
* Erik van der Poel <erik@vanderpoel.org>.
* Portions created by the Initial Developer are Copyright (C) 1998-2005
* the Initial Developer. All Rights Reserved.
*
* Contributor(s):
*
* ***** END LICENSE BLOCK ***** */
#include "all.h"
#define OUTPUT_DIRECTORY "test/robot/"
typedef struct Entry
{
int count;
unsigned char *viewURL;
unsigned char *url;
} Entry;
typedef struct Robot
{
int slot;
int count;
View *view;
URL *url;
char viewFile[1024];
int viewFileAdded;
char viewURL[1024];
} Robot;
typedef struct StatusEntry
{
time_t time;
char *message;
char *file;
int line;
} StatusEntry;
typedef struct TimeEntry
{
char *task;
int count;
double total;
double min;
double max;
} TimeEntry;
typedef void (*Handler)(int fd);
typedef struct FD
{
Handler handler;
FILE *file;
} FD;
#define numberOfSlots 64
static Thread slots[numberOfSlots];
static Thread statusThread;
static StatusEntry statusEntries[numberOfSlots];
static int sortedStatusEntries[numberOfSlots];
static unsigned short mainPort = 40404;
static int maxFD = -1;
static FD **table = NULL;
static fd_set fdSet;
static TimeEntry times[] =
{
{ "connect success", 0, 0.0, DBL_MAX, DBL_MIN },
{ "connect failure", 0, 0.0, DBL_MAX, DBL_MIN },
{ "gethostbyname_r success", 0, 0.0, DBL_MAX, DBL_MIN },
{ "gethostbyname_r failure", 0, 0.0, DBL_MAX, DBL_MIN },
{ "readStream", 0, 0.0, DBL_MAX, DBL_MIN },
{ "total", 0, 0.0, DBL_MAX, DBL_MIN }
};
#if 0
static unsigned char *limitURLs[] =
{
/*
"http://somehost/",
"http://somehost.somedomain.com/",
*/
NULL
};
#endif
static char *outFile = OUTPUT_DIRECTORY "index.html";
static char *limitDomains[16];
static int limitDomainsIndex = 0;
static int count = 0;
#define LIMIT 50
#define INTERVAL ((LIMIT / 100) ? (LIMIT / 100) : 5)
static URL *currURL = NULL;
static URL *lastURL = NULL;
static URL *urls = NULL;
static char *waiting = "waiting";
#ifdef ROBOT_LOG_ATTRIBUTES
static HashTable *attributeTable = NULL;
#endif
static HashTable *contentTypeTable = NULL;
static HashTable *httpCharsetTable = NULL;
static HashTable *httpHeaderTable = NULL;
static HashTable *metaCharsetTable = NULL;
static HashTable *schemeTable = NULL;
#ifdef ROBOT_LOG_TAGS
static HashTable *tagTable = NULL;
#endif
static HashTable *urlTable = NULL;
static HashTable *knownBadTags = NULL;
static unsigned char *badTags[] =
{
NULL
};
static FILE *statsOut = NULL;
static char *firstURL = NULL;
static unsigned char *startTime = NULL;
static void
robotStatus(App *app, char *message, char *file, int line)
{
Robot *robot;
StatusEntry *entry;
if (!app)
{
return;
}
robot = app->data;
entry = &statusEntries[robot->slot];
time(&entry->time);
entry->message = message;
entry->file = file;
entry->line = line;
}
static void
robotTime(App *app, int task, struct timeval *before)
{
struct timeval after;
double span;
gettimeofday(&after, NULL);
threadMutexLock();
span = (((after.tv_sec - before->tv_sec) * 1000000.0) +
after.tv_usec - before->tv_usec);
times[task].total += span;
if (span < times[task].min)
{
times[task].min = span;
}
if (span > times[task].max)
{
times[task].max = span;
}
times[task].count++;
threadMutexUnlock();
}
static void
addEntry(HashTable *table, Robot *robot, unsigned char *url, unsigned char *str)
{
Entry *entry;
HashEntry *hashEntry;
hashEntry = hashLookup(table, str);
if (hashEntry)
{
((Entry *) hashEntry->value)->count++;
}
else
{
entry = calloc(sizeof(Entry), 1);
if (!entry)
{
fprintf(stderr, "cannot calloc Entry\n");
exit(0);
}
entry->count = 1;
entry->viewURL = copyString((unsigned char *) robot->viewURL);
robot->viewFileAdded = 1;
entry->url = copyString(url);
hashAdd(table, copyString(str), entry);
}
}
static void
freeEntry(unsigned char *str, void *e)
{
free(str);
if (e)
{
free(((Entry *) e)->url);
free(((Entry *) e)->viewURL);
free(e);
}
}
static void
robotScheme(Robot *robot, unsigned char *url, unsigned char *scheme)
{
addEntry(schemeTable, robot, url, scheme);
}
static void
addURLFunc(App *app, URL *url)
{
Robot *robot;
if (!url->scheme)
{
return;
}
robot = app->data;
robotScheme(robot, url->url, url->scheme);
if (strcmp((char *) url->scheme, "http"))
{
urlFree(url);
}
else
{
threadMutexLock();
lastURL->next = url;
lastURL = url;
if (!currURL)
{
currURL = url;
}
threadCondSignal();
threadMutexUnlock();
}
}
static void
robotHTTP(App *app, Buf *buf)
{
viewHTTP(app, buf);
}
static void
robotHTTPBody(App *app, Buf *buf)
{
}
static void
robotHTTPHeaderName(App *app, Buf *buf)
{
Robot *robot;
unsigned char *name;
robot = app->data;
name = bufCopyLower(buf);
addEntry(httpHeaderTable, robot, robot->url->url, name);
free(name);
viewHTTPHeaderName(app, buf);
}
static void
robotHTTPHeaderValue(App *app, Buf *buf, unsigned char *url)
{
viewHTTPHeaderValue(app, buf);
}
static void
robotHTML(App *app, Buf *buf)
{
viewHTML(app, buf);
}
static void
robotHTMLText(App *app, Buf *buf)
{
Robot *robot;
unsigned char *p;
unsigned char *str;
robot = app->data;
viewHTMLText(app, buf);
str = bufCopy(buf);
p = str;
while (*p)
{
if
(
(p[0] == '&' ) &&
(p[1] == '#' ) &&
(p[2] == '1' ) &&
(p[3] == '4' ) &&
(p[4] != '\0') &&
(p[5] == ';' )
)
{
if (p[4] == '7')
{
fprintf(stderr, "147: %s\n", robot->url->url);
}
else if (p[4] == '8')
{
fprintf(stderr, "148: %s\n", robot->url->url);
}
}
p++;
}
free(str);
}
static void
robotHTMLTag(App *app, HTML *html, Buf *buf)
{
Robot *robot;
HashEntry *tagEntry;
robot = app->data;
if (html->tagIsKnown)
{
#ifdef ROBOT_LOG_TAGS
addEntry(tagTable, robot, robot->url->url, html->tag);
#endif
}
else
{
tagEntry = hashLookup(knownBadTags, html->tag);
if (!tagEntry)
{
/* XXX
printf("\t\"%s\",\n", html->tag);
*/
}
}
viewHTMLTag(app, buf);
}
static void
robotHTMLAttributeName(App *app, HTML *html, Buf *buf)
{
Robot *robot;
robot = app->data;
if (html->tagIsKnown)
{
#ifdef ROBOT_LOG_ATTRIBUTES
addEntry(attributeTable, robot, robot->url->url,
html->currentAttribute->name);
#endif
}
viewHTMLAttributeName(app, buf);
}
static void
robotHTMLAttributeValue(App *app, HTML *html, Buf *buf)
{
viewHTMLAttributeValue(app, buf);
}
static void
robotHTMLDeclaration(App *app, Buf *buf)
{
viewHTMLDeclaration(app, buf);
}
static void
robotHTMLProcessingInstruction(App *app, Buf *buf)
{
viewHTMLProcessingInstruction(app, buf);
}
static void
robotContentType(App *app, unsigned char *contentType)
{
Robot *robot;
robot = app->data;
addEntry(contentTypeTable, robot, robot->url->url, contentType);
}
static void
metaHandler(App *app, HTML *html)
{
Robot *robot;
HTMLAttribute *attr;
unsigned char *charset;
ContentType *contentType;
robot = app->data;
attr = html->attributes;
while (attr)
{
if
(
(!strcmp((char *) attr->name, "http-equiv")) &&
(attr->value) &&
(!strcasecmp((char *) attr->value, "content-type"))
)
{
break;
}
attr = attr->next;
}
if (attr)
{
contentType =
mimeParseContentType(html->currentAttribute->value);
charset = mimeGetContentTypeParameter(contentType, "charset");
mimeFreeContentType(contentType);
if (charset)
{
addEntry(metaCharsetTable, robot, robot->url->url,
lowerCase(charset));
free(charset);
}
}
}
static void
tagHandler(App *app, HTML *html)
{
}
static void
robotHTTPCharSet(App *app, unsigned char *charset)
{
Robot *robot;
robot = app->data;
addEntry(httpCharsetTable, robot, robot->url->url, charset);
}
static void
printEntry(HashEntry *hashEntry)
{
Entry *entry;
unsigned char *key;
unsigned char *url;
entry = hashEntry->value;
key = toHTML(hashEntry->key);
url = toHTML(entry->url);
fprintf
(
statsOut,
"<tr><td>%s</td><td align=right>%d</td>"
"<td><a href=%s>View Source</a></td>"
"<td><a href=%s>%s</a></td></tr>\n",
key,
entry->count,
entry->viewURL,
url,
url
);
free(key);
free(url);
}
static void
printTable(HashTable *table, char *column1)
{
fprintf(statsOut, "<table>\n");
fprintf
(
statsOut,
"<tr bgcolor=#cccccc><td align=center>%s</td><td>Count</td>"
"<td align=center>View Source</td>"
"<td align=center>Example URL</td></tr>\n",
column1
);
hashEnumerate(table, printEntry);
fprintf(statsOut, "</table>\n");
}
static void
printTimes(FILE *file)
{
int i;
fprintf(file, "<table>\n");
fprintf(file, "<tr bgcolor=#cccccc>");
fprintf(file, "<td>Task</td>");
fprintf(file, "<td>Count</td>");
fprintf(file, "<td>Average</td>");
fprintf(file, "<td>Min</td>");
fprintf(file, "<td>Max</td>");
fprintf(file, "</tr>");
for (i = 0; i < appTimeMax; i++)
{
TimeEntry *entry;
entry = &times[i];
fprintf(file, "<tr>");
fprintf(file, "<td>%s</td>", entry->task);
fprintf(file, "<td align=right>%d</td>", entry->count);
if (entry->count)
{
fprintf(file, "<td align=right>%f</td>",
(entry->total / entry->count) / 1000000);
fprintf(file, "<td align=right>%f</td>",
entry->min / 1000000);
fprintf(file, "<td align=right>%f</td>",
entry->max / 1000000);
}
else
{
fprintf(file, "<td>&nbsp;</td>");
fprintf(file, "<td>&nbsp;</td>");
fprintf(file, "<td>&nbsp;</td>");
}
fprintf(file, "</tr>");
}
fprintf(file, "</table>\n");
}
static void
printStats(void)
{
char backup[1024];
char **limit;
time_t theTime;
sprintf(backup, "%s.bak", outFile);
rename(outFile, backup);
statsOut = fopen(outFile, "w");
fprintf(statsOut, "<html><head><title>Stats</title></head><body>\n");
fprintf(statsOut, "<table bgcolor=#cccccc>\n");
fprintf(statsOut,
"<tr><td>Start Time</td><td align=right>%s</td></tr>\n",
startTime);
time(&theTime);
fprintf(statsOut,
"<tr><td>Time of This File</td><td align=right>%s</td></tr>\n",
ctime(&theTime));
fprintf(statsOut,
"<tr><td>Root URL</td><td align=right>%s</td></tr>\n",
firstURL);
fprintf(statsOut, "<tr><td>Domain Limits</td><td align=right>");
limit = limitDomains;
while (*limit)
{
fprintf(statsOut, "%s ", *limit);
limit++;
}
fprintf(statsOut, "</td></tr>\n");
fprintf(statsOut,
"<tr><td>URLs Attempted</td><td align=right>%d</td></tr>\n",
count);
fprintf(statsOut,
"<tr><td>DNS Successes</td><td align=right>%d</td></tr>\n",
netGetDNSCount());
fprintf(statsOut,
"<tr><td>Connect Successes</td><td align=right>%d</td></tr>\n",
netGetConnectCount());
fprintf(statsOut,
"<tr><td>Non-empty HTTP Responses</td>"
"<td align=right>%d</td></tr>\n",
httpGetNonEmptyHTTPResponseCount());
fprintf(statsOut,
"<tr><td>HTTP/1.0 or Greater</td>"
"<td align=right>%d</td></tr>\n",
httpGetHTTP10OrGreaterCount());
fprintf(statsOut, "</table>\n");
printTimes(statsOut);
printTable(schemeTable, "URL Scheme");
printTable(httpHeaderTable, "HTTP Header");
printTable(contentTypeTable, "Content-Type");
printTable(httpCharsetTable, "HTTP charset");
printTable(metaCharsetTable, "META charset");
#ifdef ROBOT_LOG_TAGS
printTable(tagTable, "HTML Tag");
#endif
#ifdef ROBOT_LOG_ATTRIBUTES
printTable(attributeTable, "HTML Attribute");
#endif
fprintf(statsOut, "</body></html>\n");
fclose(statsOut);
}
static void
openViewFile(App *app)
{
Robot *robot;
robot = app->data;
sprintf(robot->viewURL, "%010d.html", robot->count);
sprintf(robot->viewFile, "%s%s", OUTPUT_DIRECTORY, robot->viewURL);
/*
sprintf(robot->viewFile, "/dev/null");
*/
robot->viewFileAdded = 0;
app->view.out = fopen(robot->viewFile, "w");
if (!app->view.out)
{
fprintf(stderr, "cannot open %s for writing: %s\n",
robot->viewFile, strerror(errno));
exit(0);
}
fprintf(app->view.out,
"<html><head><title>View</title></head><body><pre>");
}
static void
closeViewFile(App *app)
{
Robot *robot;
robot = app->data;
fprintf(app->view.out, "</pre></body></html>");
fclose(app->view.out);
if (!robot->viewFileAdded)
{
unlink(robot->viewFile);
}
robot->viewFileAdded = 0;
}
static void
processURL(App *app)
{
Robot *robot;
struct timeval theTime;
robot = app->data;
gettimeofday(&theTime, NULL);
app->status(app, "processURL", __FILE__, __LINE__);
openViewFile(app);
httpFree(httpProcess(app, robot->url, NULL, NULL));
closeViewFile(app);
app->status(app, "processURL done", __FILE__, __LINE__);
app->time(app, appTimeTotal, &theTime);
}
static void *
startHere(void *a)
{
Robot robot;
App *app;
robot.slot = (int) a;
app = appAlloc();
app->status = robotStatus;
app->time = robotTime;
app->httpResponse = robotHTTP;
app->httpResponseBody = robotHTTPBody;
app->httpResponseHeaderName = robotHTTPHeaderName;
app->httpResponseHeaderValue = robotHTTPHeaderValue;
app->html = robotHTML;
app->htmlAttributeName = robotHTMLAttributeName;
app->htmlAttributeValue = robotHTMLAttributeValue;
app->htmlDeclaration = robotHTMLDeclaration;
app->htmlProcessingInstruction = robotHTMLProcessingInstruction;
app->htmlTag = robotHTMLTag;
app->htmlText = robotHTMLText;
app->contentType = robotContentType;
app->httpResponseCharSet = robotHTTPCharSet;
app->data = &robot;
while (1)
{
threadMutexLock();
while ((!currURL) && (count < LIMIT))
{
app->status(app, waiting, __FILE__, __LINE__);
threadCondWait();
}
if (count >= LIMIT)
{
threadMutexUnlock();
break;
}
count++;
if (!(count % INTERVAL))
{
printStats();
}
robot.count = count;
robot.url = currURL;
currURL = currURL->next;
threadMutexUnlock();
processURL(app);
}
return NULL;
}
static int
allThreadsAreWaiting(void)
{
int i;
for (i = 0; i < numberOfSlots; i++)
{
if (statusEntries[i].message != waiting)
{
break;
}
}
return i == numberOfSlots;
}
static void
initKnownBadTags(void)
{
unsigned char **p;
knownBadTags = hashAlloc(NULL);
p = badTags;
while (*p)
{
hashAdd(knownBadTags, *p, NULL);
p++;
}
}
static FD *
addFD(int fd, Handler func)
{
FD *f;
if (fd > maxFD)
{
if (table)
{
table = utilRealloc(table,
(maxFD + 1) * sizeof(*table),
(fd + 1) * sizeof(*table));
}
else
{
table = calloc(fd + 1, sizeof(*table));
}
if (!table)
{
return NULL;
}
maxFD = fd;
}
f = malloc(sizeof(FD));
if (!f)
{
return NULL;
}
f->handler = func;
f->file = NULL;
/*
f->id = -1;
f->port = 0;
f->suspend = 0;
f->writeFD = -1;
*/
table[fd] = f;
FD_SET(fd, &fdSet);
return f;
}
static void
removeFD(int fd)
{
FD *f;
f = table[fd];
if (f)
{
FD_CLR(fd, &fdSet);
if (f->file && (fileno(f->file) == fd))
{
fclose(f->file);
}
else
{
close(fd);
}
free(f);
table[fd] = NULL;
}
}
static int
compareStatusEntries(const void *e1, const void *e2)
{
StatusEntry *entry1;
StatusEntry *entry2;
entry1 = &statusEntries[*((int *) e1)];
entry2 = &statusEntries[*((int *) e2)];
return entry1->time - entry2->time;
}
static void
readClientRequest(int fd)
{
unsigned char b[10240];
int bytesRead;
FILE *file;
int i;
bytesRead = recv(fd, b, sizeof(b) - 1, 0);
if (bytesRead < 0)
{
if (errno != ECONNRESET)
{
perror("recv");
}
removeFD(fd);
return;
}
else if (!bytesRead)
{
removeFD(fd);
return;
}
b[bytesRead] = 0;
file = fdopen(fd, "w");
if (!file)
{
char *err = "fdopen failed\n";
send(fd, err, strlen(err), 0);
removeFD(fd);
return;
}
table[fd]->file = file;
if (strstr((char *) b, "/exit"))
{
char *goodbye =
"HTTP/1.0 200 OK\n"
"Content-Type: text/html\n"
"\n"
"Bye!"
;
fprintf(file, goodbye);
removeFD(fd);
exit(0);
}
else if (strstr((char *) b, "/times"))
{
char *begin =
"HTTP/1.0 200 OK\n"
"Content-Type: text/html\n"
"\n"
;
fprintf(file, begin);
printTimes(file);
removeFD(fd);
}
else
{
char *hello =
"HTTP/1.0 200 OK\n"
"Content-Type: text/html\n"
"\n"
;
fprintf(file, hello);
fprintf(file, "<table>\n");
fprintf(file, "<tr bgcolor=#cccccc>");
fprintf(file, "<td>Time</td>");
fprintf(file, "<td>Message</td>");
fprintf(file, "<td>File</td>");
fprintf(file, "<td>Line</td>");
fprintf(file, "</tr>");
for (i = 0; i < numberOfSlots; i++)
{
sortedStatusEntries[i] = i;
}
qsort(sortedStatusEntries, numberOfSlots, sizeof(int),
compareStatusEntries);
for (i = 0; i < numberOfSlots; i++)
{
StatusEntry *entry;
entry = &statusEntries[sortedStatusEntries[i]];
fprintf(file, "<tr>");
fprintf(file, "<td>%s</td>", ctime(&entry->time));
fprintf(file, "<td>%s</td>",
entry->message ? entry->message : "NULL");
fprintf(file, "<td>%s</td>",
entry->file ? entry->file : "NULL");
fprintf(file, "<td>%d</td>", entry->line);
fprintf(file, "</tr>");
}
fprintf(file, "</table>\n");
removeFD(fd);
}
}
static void
acceptNewClient(int fd)
{
FD *f;
int newFD;
newFD = netAccept(fd);
if (newFD < 0)
{
fprintf(stderr, "netAccept failed\n");
return;
}
f = addFD(newFD, readClientRequest);
if (!f)
{
fprintf(stderr, "addFD failed\n");
return;
}
}
static void *
startStatusFunc(void *a)
{
App *app;
FD *f;
int fd;
fd_set localFDSet;
int ret;
app = appAlloc();
fd = netListen(app, NULL, &mainPort);
if (fd < 0)
{
fprintf(stderr, "netListen failed\n");
exit(0);
}
f = addFD(fd, acceptNewClient);
if (!f)
{
fprintf(stderr, "addFD failed\n");
exit(0);
}
while (1)
{
localFDSet = fdSet;
ret = select(maxFD + 1, &localFDSet, NULL, NULL, NULL);
if (ret == -1)
{
perror("select");
}
for (fd = 0; fd <= maxFD; fd++)
{
if (FD_ISSET(fd, &localFDSet))
{
(*table[fd]->handler)(fd);
}
}
}
return NULL;
}
static void
startStatusThread(void)
{
int ret;
ret = threadCreate(&statusThread, startStatusFunc, NULL);
if (ret)
{
exit(0);
}
}
int
main(int argc, char *argv[])
{
int i;
time_t theTime;
URL *url;
int ret;
#if !defined(WINDOWS)
if (signal(SIGPIPE, SIG_IGN) == SIG_ERR)
{
fprintf(stderr, "signal failed\n");
exit(0);
}
#endif
for (i = 1; i < argc; i++)
{
if (!strcmp(argv[i], "-d"))
{
i++;
limitDomains[limitDomainsIndex] = argv[i];
limitDomainsIndex++;
}
else if (!strcmp(argv[i], "-o"))
{
i++;
outFile = argv[i];
}
else if (!strcmp(argv[i], "-s"))
{
i++;
firstURL = argv[i];
}
}
if (!netInit())
{
return 1;
}
if (!threadInit())
{
return 1;
}
time(&theTime);
startTime = copyString((unsigned char *) ctime(&theTime));
#ifdef ROBOT_LOG_ATTRIBUTES
attributeTable = hashAlloc(freeEntry);
#endif
contentTypeTable = hashAlloc(freeEntry);
httpCharsetTable = hashAlloc(freeEntry);
httpHeaderTable = hashAlloc(freeEntry);
metaCharsetTable = hashAlloc(freeEntry);
schemeTable = hashAlloc(freeEntry);
#ifdef ROBOT_LOG_TAGS
tagTable = hashAlloc(freeEntry);
#endif
urlTable = hashAlloc(NULL);
initKnownBadTags();
addURLInit(addURLFunc, NULL, limitDomains);
htmlRegister("meta", "content", metaHandler);
htmlRegisterTagHandler(tagHandler);
startStatusThread();
for (i = 0; i < numberOfSlots; i++)
{
ret = threadCreate(&slots[i], startHere, (void *) i);
if (ret)
{
exit(0);
}
}
threadYield();
if (!firstURL)
{
firstURL = "http://mozilla.org/";
}
url = urlParse((unsigned char *) firstURL);
hashAdd(urlTable, (unsigned char *) firstURL, 0);
urls = url;
lastURL = url;
currURL = url;
threadCondSignal();
threadYield();
while (1)
{
if (allThreadsAreWaiting())
{
for (i = 0; i < numberOfSlots; i++)
{
threadCancel(slots[i]);
}
break;
}
if (count >= LIMIT)
{
threadCondBroadcast();
for (i = 0; i < numberOfSlots; i++)
{
threadJoin(slots[i]);
}
threadCancel(statusThread);
break;
}
}
printStats();
#ifdef ROBOT_LOG_ATTRIBUTES
hashFree(attributeTable);
#endif
hashFree(contentTypeTable);
hashFree(httpCharsetTable);
hashFree(httpHeaderTable);
hashFree(metaCharsetTable);
hashFree(schemeTable);
#ifdef ROBOT_LOG_TAGS
hashFree(tagTable);
#endif
hashFree(urlTable);
url = urls;
while (url)
{
URL *tmp;
tmp = url;
url = url->next;
urlFree(tmp);
}
exit(0);
return 1;
}