A Question On Fedora Libcurl Performance
Thomas Dineen
tdineen at ix.netcom.com
Thu Sep 5 17:59:16 UTC 2013
Gentle People:
While I thought that this was initially a libcurl problem it may be
a DNS or other
Fedora problem. This explains the Fedora User Group post.
I am using libcurl and some of Curl Website example code as calling
routines
for a multi-platform project where a webpage is read from
finance.yahoo.com.
Everything works as expected on Solaris 10 and MS Windows, delivering
excellent performance reading a web page.
Now when I perform the exact same access on Fedora 14 the read
performance
is very slow. When the read access is executed by sending the URL there
seems
to be a pause of one or two minutes to get a response. In the end there
is always
a correct response but the delay is unacceptable.
For both Solaris 10 and Fedora 14 the native gcc is used to build
the project,
and for Windows the Mingw Cross Environment with gcc is used. There are
varying
versions of gcc used. See below.
Please note that the exact same source code and make file is used
in the build in
all three environments.
Also the libcurl version varies in all three environments. However
this morning I updated
the Fedora 14 environment to the libcurl newest version 7.32 and the
performance did not
improve.
From Rich Grey:
Could this be a server/DNS problem? I've seen long delays like this
when telnetting to a server which tries to do a reverse DNS lookup on
the connecting client and fails. After the lookup timeout, which can
be minutes, the login: prompt finally appears and the session proceeds
normally.
Please note the attached Read_Yahoo.c function that I use for the web
page read.
Please note that I can post more information if requested.
- Solaris 10
gcc -dumpversion
3.4.6
- Fedora 14
gcc -dumpversion
4.5.1
Windows Mingw Cross Environment:
gcc -dumpversion
4.5.1
Thomas Dineen
On 8/31/2013 11:57 AM, Thomas Dineen wrote:
>
>
>
>
-------------- next part --------------
/******************************************************************************/
/* */
/* TA_Bench read_yahoo.c 03/07/2013 */
/* */
/******************************************************************************/
/*****************************************************************************
*
* This example source code introduces a c library buffered I/O interface to
* URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
* rewind(). Supported functions have identical prototypes to their normal c
* lib namesakes and are preceaded by url_ .
*
* Using this code you can replace your program's fopen() with url_fopen()
* and fread() with url_fread() and it become possible to read remote streams
* instead of (only) local files. Local files (ie those that can be directly
* fopened) will drop back to using the underlying clib implementations
*
* See the main() function at the bottom that shows an app that retrives from a
* specified url using fgets() and fread() and saves as two output files.
*
* This example requires libcurl 7.9.7 or later.
*/
#include <stdio.h>
#include <string.h>
#ifndef WIN32
# include <sys/time.h>
#endif
#include <stdlib.h>
#include <errno.h>
#include <curl/curl.h>
/******************************************************************************/
/* */
/* Structures And Declarations. */
/* */
/******************************************************************************/
enum fcurl_type_e
{
CFTYPE_NONE=0,
CFTYPE_FILE=1,
CFTYPE_CURL=2
};
struct fcurl_data
{
enum fcurl_type_e type; /* type of handle */
union
{
CURL *curl;
FILE *file;
} handle; /* handle */
char *buffer; /* buffer to store cached data*/
size_t buffer_len; /* currently allocated buffers length */
size_t buffer_pos; /* end of data in buffer*/
int still_running; /* Is background url fetch still in progress */
};
typedef struct fcurl_data URL_FILE;
/* Exported Functions */
URL_FILE *url_fopen(const char *url,const char *operation);
int url_fclose(URL_FILE *file);
int url_feof(URL_FILE *file);
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
char * url_fgets(char *ptr, size_t size, URL_FILE *file);
void url_rewind(URL_FILE *file);
/* we use a global one for convenience */
CURLM *multi_handle;
// url="http://192.168.7.3/testfile";/* default to testurl */
// url="DineenConsulting.com";/* default to testurl */
// url="http://finance.yahoo.com/q/hp?s=SPY+Historical+Prices";
/* default to testurl */
/******************************************************************************/
/* */
/* Function: Read_Yahoo_Primary */
/* */
/******************************************************************************/
int Read_Yahoo_Primary ( Stock_Data_Struct *Stock_Data_Head, Stock_Div_Struct *Stock_Div_Head, Stock_Data_Info_Struct *Stock_Info_Head, Date_Struct Date_Start, Date_Struct Date_End, char Symbol[10], char Type, Print_Struct *Print_Head )
{
URL_FILE *handle;
FILE *outf;
FILE *Test_Handle;
int nread;
char buffer[256]; // Was 256
char Char_Temp[16];
const char *url;
char URL_String[256];
char Temp[256];
int year;
int month;
int day;
float open_read;
float high_read;
float low_read;
float close_read;
long long int volume_read;
float adj_close_read;
float open;
float high;
float low;
float close;
long long int volume;
float adj_close;
float dividend;
Stock_Data_Struct *curr_ptr;
Stock_Data_Struct *new_ptr;
Stock_Data_Struct *prev_ptr;
Stock_Div_Struct *div_curr_ptr;
Stock_Div_Struct *div_new_ptr;
Stock_Div_Struct *div_prev_ptr;
int First_Record;
int Entry_Count;
float Price_High;
float Price_Low;
float Split_Divisor;
float Divisor;
float Int_Divisor;
/******************************************************************************/
/* */
/* Print The Input Variables: */
/* */
/******************************************************************************/
if ( Print_Head->Print_Headers_Read == TRUE )
{
printf ( "Running Read_Yahoo\n") ;
printf ( "Start Date: Month = %d\n", Date_Start.Month );
printf ( "Start Date: Day = %d\n", Date_Start.Day );
printf ( "Start Date: Year = %d\n", Date_Start.Year );
printf ( "End Date: Month = %d\n", Date_End.Month );
printf ( "End Date: Day = %d\n", Date_End.Day );
printf ( "End Date: Year = %d\n", Date_End.Year );
printf ( "Symbol = %s\n", Symbol );
printf ( "Type = %c\n", Type );
}
/******************************************************************************/
/* */
/* Assemble The URL: */
/* */
/******************************************************************************/
strcpy ( URL_String, "http://ichart.finance.yahoo.com/table.csv?s=" );
strcat ( URL_String, Symbol );
strcat ( URL_String, "&a=" );
sprintf ( Temp, "%d", Date_Start.Month-1);
strcat ( URL_String, Temp );
strcat ( URL_String, "&b=" );
sprintf ( Temp, "%d", Date_Start.Day );
strcat ( URL_String, Temp );
strcat ( URL_String, "&c=" );
sprintf ( Temp, "%d", Date_Start.Year);
strcat ( URL_String, Temp );
strcat ( URL_String, "&g=" );
sprintf ( Temp, "%c", Type );
strcat ( URL_String, Temp );
strcat ( URL_String, "&d=" );
sprintf ( Temp, "%d", Date_End.Month-1);
strcat ( URL_String, Temp );
strcat ( URL_String, "&e=" );
sprintf ( Temp, "%d", Date_End.Day);
strcat ( URL_String, Temp );
strcat ( URL_String, "&f=" );
sprintf ( Temp, "%d", Date_End.Year);
strcat ( URL_String, Temp );
strcat ( URL_String, "&ignore=.csv" );
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "URL String = %s\n", URL_String );
/* url="http://ichart.finance.yahoo.com/table.csv?s=SPY&d=10&e=2&f=2012&g=d&a=0&b=29&c=1993&ignore=.csv"; */
/* url="http://ichart.finance.yahoo.com/table.csv?s=SPY&d=10&e=2&f=2012&g=d&a=0&b=29&c=1993&ignore=.csv"; */
/******************************************************************************/
/* */
/* Read The Data From The Yahoo Web Site: */
/* */
/******************************************************************************/
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Yahoo Web Site Read\n" );
handle = url_fopen ( URL_String, "r" );
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Handle = %d\n", handle );
if( !handle )
{
printf( "Couldn't Open URL: url_fopen() %s\n", url );
printf ( "Handle = %d\n", handle );
fclose( outf );
return -1;
}
/******************************************************************************/
/* */
/* Read The Price / Volume Data: */
/* */
/******************************************************************************/
if ( Type == 'd' || Type == 'w' || Type == 'm' )
{ /* Start Type Price Data Block */
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Price Volume Read\n" );
curr_ptr = Stock_Data_Head;
prev_ptr = Stock_Data_Head;
prev_ptr->high = 1000000.0;
First_Record = TRUE;
Entry_Count = 0;
Price_High = 0.0;
Price_Low = 1000000000.0;
Split_Divisor = 1.0;
Divisor = 0.0;
Int_Divisor = 1.0;
while( !url_feof( handle ))
{ /* Start While */
url_fgets( buffer, sizeof( buffer ), handle);
if ( strstr ( buffer, "404 Not Found" ) != NULL )
{
printf( "Error 404 Symbol Not Found\n" );
fclose( outf );
return ( -1 );
}
if ( Print_Head->Print_Data == TRUE )
printf ( "%s", buffer );
if ( First_Record == FALSE )
{ /* Start Not First_Record */
sscanf ( buffer, "%4d", &year );
sscanf ( buffer+5, "%2d", &month );
sscanf ( buffer+8, "%2d", &day );
sscanf ( buffer+11, "%f,%f,%f,%f,%lli,%f", &open_read, &high_read,
&low_read, &close_read, &volume_read, &adj_close_read );
if ( ( 1.40 * Split_Divisor * prev_ptr->high ) <= high_read )
{ /* Start Block Split Occured */
Divisor = ( high_read / (Split_Divisor*prev_ptr->high));
if ( Divisor >= 7.8 )
Int_Divisor = 8;
else
if ( Divisor >= 6.8 )
Int_Divisor = 7;
else
if ( Divisor >= 5.8 )
Int_Divisor = 6;
else
if ( Divisor >= 4.8 )
Int_Divisor = 5;
else
if ( Divisor >= 3.8 )
Int_Divisor = 4;
else
if ( Divisor >= 2.8 )
Int_Divisor = 3;
else
if ( Divisor >= 1.6 )
Int_Divisor = 2;
else
if ( Divisor >= 1.3 )
Int_Divisor = 1.5;
Split_Divisor = (float) Split_Divisor * Int_Divisor;
curr_ptr->Split_Occured = TRUE;
if ( Print_Head->Print_Data == TRUE )
printf ( "Split Detected: Date: %d/%d/%d Int_Divisor %d Split_Divisor %f\n", month, day, year, Int_Divisor, Split_Divisor);
} /* End Block Split Occured */
else
{ /* Start Block No Split Occured */
Int_Divisor = 0.0;
curr_ptr->Split_Occured = FALSE;
}; /* End Block No Split Occured */
open = open_read / Split_Divisor;
high = high_read / Split_Divisor;
low = low_read / Split_Divisor;
close = close_read / Split_Divisor;
volume = volume_read;
adj_close = adj_close_read / Split_Divisor;
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Curr_Ptr %x\n", curr_ptr );
printf ( "Date: %4i %2i %2i\n", year, month, day );
printf ( "Open: %2.2f\n", open);
printf ( "High: %2.2f\n", high );
printf ( "Low: %2.2f\n", low );
printf ( "Close: %2.2f\n", close );
printf ( "Volume: %i\n", volume );
printf ( "Adj_Close: %2.2f\n", adj_close );
printf ( "High_Read: %2.2f\n", high_read);
printf ( "Prev High: %2.2f\n", prev_ptr->high );
printf ( "Divisor: %2.2f\n", Divisor );
printf ( "Split_Divisor: %2.2f\n", Split_Divisor );
printf ( "Int_Divisor: %2.2f\n\n", Int_Divisor );
};
curr_ptr->year = year;
curr_ptr->month = month;
curr_ptr->day = day;
curr_ptr->open = open;
curr_ptr->high = high;
curr_ptr->low = low;
curr_ptr->close = close;
curr_ptr->volume = volume;
curr_ptr->adj_close = adj_close;
curr_ptr->open_read = open_read;
curr_ptr->high_read = high_read;
curr_ptr->low_read = low_read;
curr_ptr->close_read = close_read;
curr_ptr->volume_read = volume_read;
curr_ptr->adj_close_read = adj_close_read;
curr_ptr->Int_Divisor = Int_Divisor;
curr_ptr->Split_Divisor = Split_Divisor;
curr_ptr->Entry_Count = Entry_Count;
if ( high > Price_High )
Price_High = high;
if ( low < Price_Low )
Price_Low = low;
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Price_High = %f\n", Price_High );
printf ( "Price_Low = %f\n", Price_Low );
};
new_ptr = malloc( sizeof( Stock_Data_Struct) );
curr_ptr->next_ptr = new_ptr;
curr_ptr->prev_ptr = prev_ptr;
prev_ptr = curr_ptr;
curr_ptr = new_ptr;
Entry_Count++;
}; /* End Not First_Record */
First_Record = FALSE;
}; /* End While */
if ( Entry_Count != NULL )
curr_ptr = prev_ptr;
curr_ptr->next_ptr = NULL;
Stock_Info_Head->Date_Start.Day = Date_Start.Day;
Stock_Info_Head->Date_Start.Month = Date_Start.Month;
Stock_Info_Head->Date_Start.Year= Date_Start.Year;
Stock_Info_Head->Date_End.Day = Date_End.Day;
Stock_Info_Head->Date_End.Month = Date_End.Month;
Stock_Info_Head->Date_End.Year = Date_End.Year;
strcpy ( Stock_Info_Head->Symbol, Symbol );
Stock_Info_Head->Type = Type;
Stock_Info_Head->Entry_Count = Entry_Count;
Stock_Info_Head->Price_High = Price_High;
Stock_Info_Head->Price_Low = Price_Low;
Stock_Info_Head->First_Ptr = Stock_Data_Head;
Stock_Info_Head->Last_Ptr = curr_ptr;
url_fclose(handle);
return (NULL); /* all done */
} /* End Type Price Data Block */
else
/******************************************************************************/
/* */
/* Read The Dividend Data: */
/* */
/******************************************************************************/
if ( Type == 'v' )
{ /* Start Type Dividend Data Block */
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Dividend Read\n" );
div_curr_ptr = Stock_Div_Head;
div_prev_ptr = NULL;
First_Record = TRUE;
Entry_Count = 0;
while( !url_feof( handle ))
{ /* Start While */
url_fgets( buffer, sizeof( buffer ), handle);
if ( Print_Head->Print_Data == TRUE )
printf ( "%s", buffer );
if ( First_Record == FALSE )
{ /* Start Not First_Record */
sscanf ( buffer, "%4d", &year );
sscanf ( buffer+5, "%2d", &month );
sscanf ( buffer+8, "%2d", &day );
sscanf ( buffer+11, "%f", ÷nd );
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Curr_Ptr %x\n", div_curr_ptr );
printf ( "Date: %4i %2i %2i\n", year, month, day );
printf ( "Dividend %2.2f\n", dividend );
};
div_curr_ptr->year = year;
div_curr_ptr->month = month;
div_curr_ptr->day = day;
div_curr_ptr->Dividend = dividend;
div_new_ptr = malloc( sizeof( Stock_Div_Struct) );
div_curr_ptr->next_ptr = div_new_ptr;
div_curr_ptr->prev_ptr = div_prev_ptr;
div_prev_ptr = div_curr_ptr;
div_curr_ptr = div_new_ptr;
Entry_Count++;
}; /* End Not First_Record */
First_Record = FALSE;
}; /* End While */
if ( Entry_Count != 0 )
div_curr_ptr = div_prev_ptr;
div_curr_ptr->next_ptr = NULL;
Stock_Info_Head->Date_Start.Day = Date_Start.Day;
Stock_Info_Head->Date_Start.Month = Date_Start.Month;
Stock_Info_Head->Date_Start.Year= Date_Start.Year;
Stock_Info_Head->Date_End.Day = Date_End.Day;
Stock_Info_Head->Date_End.Month = Date_End.Month;
Stock_Info_Head->Date_End.Year = Date_End.Year;
strcpy ( Stock_Info_Head->Symbol, Symbol );
Stock_Info_Head->Type = Type;
Stock_Info_Head->Entry_Count = Entry_Count;
Stock_Info_Head->First_Ptr = Stock_Div_Head;
Stock_Info_Head->Last_Ptr = div_curr_ptr;
url_fclose(handle);
return (NULL); /* all done */
} /* End Type Dividend Data Block */
};
/******************************************************************************/
/* */
/* Function: Read_Yahoo_Secondary */
/* */
/******************************************************************************/
int Read_Yahoo_Secondary ( Stock_Data_Struct *Stock_Data_Head, Stock_Div_Struct *Stock_Div_Head, Stock_Data_Info_Struct *Stock_Info_Head, Date_Struct Date_Start, Date_Struct Date_End, char Symbol[10], char Type, Print_Struct *Print_Head )
{
URL_FILE *handle;
FILE *outf;
FILE *Test_Handle;
int nread;
char buffer[256]; // Was 256
char Char_Temp[16];
const char *url;
char URL_String[256];
char Temp[256];
int year;
int month;
int day;
float open_read;
float high_read;
float low_read;
float close_read;
long long int volume_read;
float adj_close_read;
float open;
float high;
float low;
float close;
long long int volume;
float adj_close;
float dividend;
Stock_Data_Struct *curr_ptr;
Stock_Data_Struct *new_ptr;
Stock_Data_Struct *prev_ptr;
Stock_Div_Struct *div_curr_ptr;
Stock_Div_Struct *div_new_ptr;
Stock_Div_Struct *div_prev_ptr;
int First_Record;
int Entry_Count;
float Price_High;
float Price_Low;
float Split_Divisor;
float Divisor;
float Int_Divisor;
/******************************************************************************/
/* */
/* Print The Input Variables: */
/* */
/******************************************************************************/
if ( Print_Head->Print_Headers_Read == TRUE )
{
printf ( "Running Read_Yahoo\n") ;
printf ( "Start Date: Month = %d\n", Date_Start.Month );
printf ( "Start Date: Day = %d\n", Date_Start.Day );
printf ( "Start Date: Year = %d\n", Date_Start.Year );
printf ( "End Date: Month = %d\n", Date_End.Month );
printf ( "End Date: Day = %d\n", Date_End.Day );
printf ( "End Date: Year = %d\n", Date_End.Year );
printf ( "Symbol = %s\n", Symbol );
printf ( "Type = %c\n", Type );
}
/******************************************************************************/
/* */
/* Assemble The URL: */
/* */
/******************************************************************************/
strcpy ( URL_String, "http://ichart.finance.yahoo.com/table.csv?s=" );
strcat ( URL_String, Symbol );
strcat ( URL_String, "&a=" );
sprintf ( Temp, "%d", Date_Start.Month-1);
strcat ( URL_String, Temp );
strcat ( URL_String, "&b=" );
sprintf ( Temp, "%d", Date_Start.Day );
strcat ( URL_String, Temp );
strcat ( URL_String, "&c=" );
sprintf ( Temp, "%d", Date_Start.Year);
strcat ( URL_String, Temp );
strcat ( URL_String, "&g=" );
sprintf ( Temp, "%c", Type );
strcat ( URL_String, Temp );
strcat ( URL_String, "&d=" );
sprintf ( Temp, "%d", Date_End.Month-1);
strcat ( URL_String, Temp );
strcat ( URL_String, "&e=" );
sprintf ( Temp, "%d", Date_End.Day);
strcat ( URL_String, Temp );
strcat ( URL_String, "&f=" );
sprintf ( Temp, "%d", Date_End.Year);
strcat ( URL_String, Temp );
strcat ( URL_String, "&ignore=.csv" );
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "URL String = %s\n", URL_String );
/* url="http://ichart.finance.yahoo.com/table.csv?s=SPY&d=10&e=2&f=2012&g=d&a=0&b=29&c=1993&ignore=.csv"; */
/* url="http://ichart.finance.yahoo.com/table.csv?s=SPY&d=10&e=2&f=2012&g=d&a=0&b=29&c=1993&ignore=.csv"; */
/******************************************************************************/
/* */
/* Read The Data From The Yahoo Web Site: */
/* */
/******************************************************************************/
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Yahoo Web Site Read\n" );
handle = url_fopen ( URL_String, "r" );
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Handle = %d\n", handle );
if( !handle )
{
printf( "Couldn't Open URL: url_fopen() %s\n", url );
printf ( "Handle = %d\n", handle );
fclose( outf );
return -1;
}
/******************************************************************************/
/* */
/* Read The Price / Volume Data: */
/* */
/******************************************************************************/
if ( Type == 'd' || Type == 'w' || Type == 'm' )
{ /* Start Type Price Data Block */
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Price Volume Read\n" );
curr_ptr = Stock_Data_Head;
prev_ptr = Stock_Data_Head;
prev_ptr->high = 1000000.0;
First_Record = TRUE;
Entry_Count = 0;
Price_High = 0.0;
Price_Low = 1000000000.0;
Split_Divisor = 1.0;
Divisor = 0.0;
Int_Divisor = 0.0;
while( !url_feof( handle ))
{ /* Start While */
url_fgets( buffer, sizeof( buffer ), handle);
if ( strstr ( buffer, "404 Not Found" ) != NULL )
{
printf( "Error 404 Symbol Not Found\n" );
fclose( outf );
return ( -1 );
}
if ( Print_Head->Print_Data == TRUE )
printf ( "%s", buffer );
if ( First_Record == FALSE )
{ /* Start Not First_Record */
sscanf ( buffer, "%4d", &year );
sscanf ( buffer+5, "%2d", &month );
sscanf ( buffer+8, "%2d", &day );
sscanf ( buffer+11, "%f,%f,%f,%f,%lli,%f", &open_read, &high_read,
&low_read, &close_read, &volume_read, &adj_close_read );
if ( ( 1.40 * Split_Divisor * prev_ptr->high ) <= high_read )
{ /* Start Block Split Occured */
Divisor = ( high_read / (Split_Divisor*prev_ptr->high));
if ( Divisor >= 7.8 )
Int_Divisor = 8;
else
if ( Divisor >= 6.8 )
Int_Divisor = 7;
else
if ( Divisor >= 5.8 )
Int_Divisor = 6;
else
if ( Divisor >= 4.8 )
Int_Divisor = 5;
else
if ( Divisor >= 3.8 )
Int_Divisor = 4;
else
if ( Divisor >= 2.8 )
Int_Divisor = 3;
else
if ( Divisor >= 1.6 )
Int_Divisor = 2;
else
if ( Divisor >= 1.3 )
Int_Divisor = 1.5;
Split_Divisor = (float) Split_Divisor * Int_Divisor;
curr_ptr->Split_Occured = TRUE;
if ( Print_Head->Print_Data == TRUE )
printf ( "Split Detected: Date: %d/%d/%d Int_Divisor %d Split_Divisor %f\n", month, day, year, Int_Divisor, Split_Divisor);
} /* End Block Split Occured */
else
{ /* Start Block No Split Occured */
Int_Divisor = 0.0;
curr_ptr->Split_Occured = FALSE;
}; /* End Block No Split Occured */
open = open_read / Split_Divisor;
high = high_read / Split_Divisor;
low = low_read / Split_Divisor;
close = close_read / Split_Divisor;
volume = volume_read;
adj_close = adj_close_read / Split_Divisor;
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Curr_Ptr %x\n", curr_ptr );
printf ( "Date: %4i %2i %2i\n", year, month, day );
printf ( "Open: %2.2f\n", open);
printf ( "High: %2.2f\n", high );
printf ( "Low: %2.2f\n", low );
printf ( "Close: %2.2f\n", close );
printf ( "Volume: %i\n", volume );
printf ( "Adj_Close: %2.2f\n", adj_close );
printf ( "High_Read: %2.2f\n", high_read);
printf ( "Prev High: %2.2f\n", prev_ptr->high );
printf ( "Divisor: %2.2f\n", Divisor );
printf ( "Split_Divisor: %2.2f\n", Split_Divisor );
printf ( "Int_Divisor: %2.2f\n\n", Int_Divisor );
};
curr_ptr->year = year;
curr_ptr->month = month;
curr_ptr->day = day;
curr_ptr->open = open;
curr_ptr->high = high;
curr_ptr->low = low;
curr_ptr->close = close;
curr_ptr->volume = volume;
curr_ptr->adj_close = adj_close;
curr_ptr->open_read = open_read;
curr_ptr->high_read = high_read;
curr_ptr->low_read = low_read;
curr_ptr->close_read = close_read;
curr_ptr->volume_read = volume_read;
curr_ptr->adj_close_read = adj_close_read;
curr_ptr->Int_Divisor = Int_Divisor;
curr_ptr->Split_Divisor = Split_Divisor;
curr_ptr->Entry_Count = Entry_Count;
if ( high > Price_High )
Price_High = high;
if ( low < Price_Low )
Price_Low = low;
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Price_High = %f\n", Price_High );
printf ( "Price_Low = %f\n", Price_Low );
};
new_ptr = malloc( sizeof( Stock_Data_Struct) );
curr_ptr->next_ptr = new_ptr;
curr_ptr->prev_ptr = prev_ptr;
prev_ptr = curr_ptr;
curr_ptr = new_ptr;
Entry_Count++;
}; /* End Not First_Record */
First_Record = FALSE;
}; /* End While */
if ( Entry_Count != NULL )
curr_ptr = prev_ptr;
curr_ptr->next_ptr = NULL;
Stock_Info_Head->Date_Start.Day = Date_Start.Day;
Stock_Info_Head->Date_Start.Month = Date_Start.Month;
Stock_Info_Head->Date_Start.Year= Date_Start.Year;
Stock_Info_Head->Date_End.Day = Date_End.Day;
Stock_Info_Head->Date_End.Month = Date_End.Month;
Stock_Info_Head->Date_End.Year = Date_End.Year;
strcpy ( Stock_Info_Head->Symbol, Symbol );
Stock_Info_Head->Type = Type;
Stock_Info_Head->Entry_Count = Entry_Count;
Stock_Info_Head->Price_High = Price_High;
Stock_Info_Head->Price_Low = Price_Low;
Stock_Info_Head->First_Ptr = Stock_Data_Head;
Stock_Info_Head->Last_Ptr = curr_ptr;
url_fclose(handle);
return (NULL); /* all done */
} /* End Type Price Data Block */
else
/******************************************************************************/
/* */
/* Read The Dividend Data: */
/* */
/******************************************************************************/
if ( Type == 'v' )
{ /* Start Type Dividend Data Block */
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Dividend Read\n" );
div_curr_ptr = Stock_Div_Head;
div_prev_ptr = NULL;
First_Record = TRUE;
Entry_Count = 0;
while( !url_feof( handle ))
{ /* Start While */
url_fgets( buffer, sizeof( buffer ), handle);
if ( Print_Head->Print_Data == TRUE )
printf ( "%s", buffer );
if ( First_Record == FALSE )
{ /* Start Not First_Record */
sscanf ( buffer, "%4d", &year );
sscanf ( buffer+5, "%2d", &month );
sscanf ( buffer+8, "%2d", &day );
sscanf ( buffer+11, "%f", ÷nd );
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Curr_Ptr %x\n", div_curr_ptr );
printf ( "Date: %4i %2i %2i\n", year, month, day );
printf ( "Dividend %2.2f\n", dividend );
};
div_curr_ptr->year = year;
div_curr_ptr->month = month;
div_curr_ptr->day = day;
div_curr_ptr->Dividend = dividend;
div_new_ptr = malloc( sizeof( Stock_Div_Struct) );
div_curr_ptr->next_ptr = div_new_ptr;
div_curr_ptr->prev_ptr = div_prev_ptr;
div_prev_ptr = div_curr_ptr;
div_curr_ptr = div_new_ptr;
Entry_Count++;
}; /* End Not First_Record */
First_Record = FALSE;
}; /* End While */
if ( Entry_Count != 0 )
div_curr_ptr = div_prev_ptr;
div_curr_ptr->next_ptr = NULL;
Stock_Info_Head->Date_Start.Day = Date_Start.Day;
Stock_Info_Head->Date_Start.Month = Date_Start.Month;
Stock_Info_Head->Date_Start.Year= Date_Start.Year;
Stock_Info_Head->Date_End.Day = Date_End.Day;
Stock_Info_Head->Date_End.Month = Date_End.Month;
Stock_Info_Head->Date_End.Year = Date_End.Year;
strcpy ( Stock_Info_Head->Symbol, Symbol );
Stock_Info_Head->Type = Type;
Stock_Info_Head->Entry_Count = Entry_Count;
Stock_Info_Head->First_Ptr = Stock_Div_Head;
Stock_Info_Head->Last_Ptr = div_curr_ptr;
url_fclose(handle);
return (NULL); /* all done */
} /* End Type Dividend Data Block */
};
/******************************************************************************/
/* */
/* Function: write_callback */
/* */
/******************************************************************************/
/* curl calls this routine to get more data */
static size_t write_callback(char *buffer,
size_t size,
size_t nitems,
void *userp)
{
char *newbuff;
size_t rembuff;
URL_FILE *url = (URL_FILE *)userp;
size *= nitems;
rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */
if(size > rembuff) {
/* not enough space in buffer */
newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
if(newbuff==NULL) {
fprintf(stderr,"callback buffer grow failed\n");
size=rembuff;
}
else {
/* realloc suceeded increase buffer size*/
url->buffer_len+=size - rembuff;
url->buffer=newbuff;
}
}
memcpy(&url->buffer[url->buffer_pos], buffer, size);
url->buffer_pos += size;
return size;
}
/* use to attempt to fill the read buffer up to requested number of bytes */
/******************************************************************************/
/* */
/* Function: fill_buffer */
/* */
/******************************************************************************/
static int fill_buffer(URL_FILE *file, size_t want)
{
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
struct timeval timeout;
int rc;
/* only attempt to fill buffer if transactions still running and buffer
* doesnt exceed required size already
*/
if((!file->still_running) || (file->buffer_pos > want))
return 0;
/* attempt to fill buffer */
do {
int maxfd = -1;
long curl_timeo = -1;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
/* set a suitable timeout to fail on */
timeout.tv_sec = 60; /* 1 minute */
timeout.tv_usec = 0;
curl_multi_timeout(multi_handle, &curl_timeo);
if(curl_timeo >= 0) {
timeout.tv_sec = curl_timeo / 1000;
if(timeout.tv_sec > 1)
timeout.tv_sec = 1;
else
timeout.tv_usec = (curl_timeo % 1000) * 1000;
}
/* get file descriptors from the transfers */
curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
/* In a real-world program you OF COURSE check the return code of the
function calls. On success, the value of maxfd is guaranteed to be
greater or equal than -1. We call select(maxfd + 1, ...), specially
in case of (maxfd == -1), we call select(0, ...), which is basically
equal to sleep. */
rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
switch(rc) {
case -1:
/* select error */
break;
case 0:
default:
/* timeout or readable/writable sockets */
curl_multi_perform(multi_handle, &file->still_running);
break;
}
} while(file->still_running && (file->buffer_pos < want));
return 1;
}
/* use to remove want bytes from the front of a files buffer */
/******************************************************************************/
/* */
/* Function: use_buffer */
/* */
/******************************************************************************/
static int use_buffer(URL_FILE *file,int want)
{
/* sort out buffer */
if((file->buffer_pos - want) <=0) {
/* ditch buffer - write will recreate */
if(file->buffer)
free(file->buffer);
file->buffer=NULL;
file->buffer_pos=0;
file->buffer_len=0;
}
else {
/* move rest down make it available for later */
memmove(file->buffer,
&file->buffer[want],
(file->buffer_pos - want));
file->buffer_pos -= want;
}
return 0;
}
URL_FILE *url_fopen(const char *url,const char *operation)
{
/* this code could check for URLs or types in the 'url' and
basicly use the real fopen() for standard files */
URL_FILE *file;
(void)operation;
file = malloc(sizeof(URL_FILE));
if(!file)
return NULL;
memset(file, 0, sizeof(URL_FILE));
if((file->handle.file=fopen(url,operation)))
file->type = CFTYPE_FILE; /* marked as URL */
else
{
file->type = CFTYPE_CURL; /* marked as URL */
file->handle.curl = curl_easy_init();
curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
if(!multi_handle)
multi_handle = curl_multi_init();
curl_multi_add_handle(multi_handle, file->handle.curl);
/* lets start the fetch */
curl_multi_perform(multi_handle, &file->still_running);
if((file->buffer_pos == 0) && (!file->still_running)) {
/* if still_running is 0 now, we should return NULL */
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(multi_handle, file->handle.curl);
/* cleanup */
curl_easy_cleanup(file->handle.curl);
free(file);
file = NULL;
}
}
return file;
}
/******************************************************************************/
/* */
/* Function: url_fclose */
/* */
/******************************************************************************/
int url_fclose(URL_FILE *file)
{
int ret=0;/* default is good return */
switch(file->type) {
case CFTYPE_FILE:
ret=fclose(file->handle.file); /* passthrough */
break;
case CFTYPE_CURL:
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(multi_handle, file->handle.curl);
/* cleanup */
curl_easy_cleanup(file->handle.curl);
break;
default: /* unknown or supported type - oh dear */
ret=EOF;
errno=EBADF;
break;
}
if(file->buffer)
free(file->buffer);/* free any allocated buffer space */
free(file);
return ret;
}
/******************************************************************************/
/* */
/* Function: url_feof */
/* */
/******************************************************************************/
int url_feof(URL_FILE *file)
{
int ret=0;
switch(file->type) {
case CFTYPE_FILE:
ret=feof(file->handle.file);
break;
case CFTYPE_CURL:
if((file->buffer_pos == 0) && (!file->still_running))
ret = 1;
break;
default: /* unknown or supported type - oh dear */
ret=-1;
errno=EBADF;
break;
}
return ret;
}
/******************************************************************************/
/* */
/* Function: url_fread */
/* */
/******************************************************************************/
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
{
size_t want;
switch(file->type) {
case CFTYPE_FILE:
want=fread(ptr,size,nmemb,file->handle.file);
break;
case CFTYPE_CURL:
want = nmemb * size;
fill_buffer(file,want);
/* check if theres data in the buffer - if not fill_buffer()
* either errored or EOF */
if(!file->buffer_pos)
return 0;
/* ensure only available data is considered */
if(file->buffer_pos < want)
want = file->buffer_pos;
/* xfer data to caller */
memcpy(ptr, file->buffer, want);
use_buffer(file,want);
want = want / size; /* number of items */
break;
default: /* unknown or supported type - oh dear */
want=0;
errno=EBADF;
break;
}
return want;
}
/******************************************************************************/
/* */
/* Function: url_fgets */
/* */
/******************************************************************************/
char *url_fgets(char *ptr, size_t size, URL_FILE *file)
{
size_t want = size - 1;/* always need to leave room for zero termination */
size_t loop;
switch(file->type) {
case CFTYPE_FILE:
ptr = fgets(ptr,size,file->handle.file);
break;
case CFTYPE_CURL:
fill_buffer(file,want);
/* check if theres data in the buffer - if not fill either errored or
* EOF */
if(!file->buffer_pos)
return NULL;
/* ensure only available data is considered */
if(file->buffer_pos < want)
want = file->buffer_pos;
/*buffer contains data */
/* look for newline or eof */
for(loop=0;loop < want;loop++) {
if(file->buffer[loop] == '\n') {
want=loop+1;/* include newline */
break;
}
}
/* xfer data to caller */
memcpy(ptr, file->buffer, want);
ptr[want]=0;/* allways null terminate */
use_buffer(file,want);
break;
default: /* unknown or supported type - oh dear */
ptr=NULL;
errno=EBADF;
break;
}
return ptr;/*success */
}
/******************************************************************************/
/* */
/* Function: url_rewind */
/* */
/******************************************************************************/
void url_rewind(URL_FILE *file)
{
switch(file->type) {
case CFTYPE_FILE:
rewind(file->handle.file); /* passthrough */
break;
case CFTYPE_CURL:
/* halt transaction */
curl_multi_remove_handle(multi_handle, file->handle.curl);
/* restart */
curl_multi_add_handle(multi_handle, file->handle.curl);
/* ditch buffer - write will recreate - resets stream pos*/
if(file->buffer)
free(file->buffer);
file->buffer=NULL;
file->buffer_pos=0;
file->buffer_len=0;
break;
default: /* unknown or supported type - oh dear */
break;
}
}
More information about the users
mailing list