diff options
author | Kostyantyn Ovechko <fastinetserver@gmail.com> | 2010-07-14 05:59:00 +0300 |
---|---|---|
committer | Kostyantyn Ovechko <fastinetserver@gmail.com> | 2010-07-14 05:59:00 +0300 |
commit | ff1e371e2342379a2377789b1cda82a21c0eee1f (patch) | |
tree | 893558eda33611b927b03a8c476f4d0398f3b851 /segget | |
parent | Add section [ui_server] to segget.conf file (diff) | |
download | idfetch-ff1e371e2342379a2377789b1cda82a21c0eee1f.tar.gz idfetch-ff1e371e2342379a2377789b1cda82a21c0eee1f.tar.bz2 idfetch-ff1e371e2342379a2377789b1cda82a21c0eee1f.zip |
Add to segget.conf file section [provide_proxy_fetcher_to_others].
NOTE: Proxy-fetcher downloads distfiles requested by other segget daemons.
File will be ignored if it's already downloaded.
TO-DO: Proxy-fetcher needs to check if distfile is already in the download list.
[provide_proxy_fetcher_to_others]
PROVIDE_PROXY_FETCHER_IP
Define an ip address segget will use to provide access for tuiclients.
The parameter should be a string holding your host dotted IP address.
Default:
provide_proxy_fetcher_ip=127.0.0.1
PROVIDE_PROXY_FETCHER_PORT
Define a port segget will use to provide access for tuiclients.
The parameter should be an integer.
Minimum value: 1
Maximum value: 65535
Default:
provide_proxy_fetcher_port=9777
Diffstat (limited to 'segget')
-rw-r--r-- | segget/distfile.cpp | 184 | ||||
-rw-r--r-- | segget/distfile.h | 6 | ||||
-rw-r--r-- | segget/network0.conf | 2 | ||||
-rw-r--r-- | segget/pkg.cpp | 36 | ||||
-rw-r--r-- | segget/pkg.h | 17 | ||||
-rw-r--r-- | segget/proxyfetcher.cpp | 105 | ||||
-rw-r--r-- | segget/proxyfetcher.h | 45 | ||||
-rw-r--r-- | segget/segget.conf | 53 | ||||
-rw-r--r-- | segget/segget.cpp | 67 | ||||
-rw-r--r-- | segget/segget.h | 4 | ||||
-rw-r--r-- | segget/settings.cpp | 7 | ||||
-rw-r--r-- | segget/settings.h | 12 | ||||
-rw-r--r-- | segget/ui_server.cpp | 4 |
13 files changed, 425 insertions, 117 deletions
diff --git a/segget/distfile.cpp b/segget/distfile.cpp index 9f0bb0e..506f8d8 100644 --- a/segget/distfile.cpp +++ b/segget/distfile.cpp @@ -41,7 +41,7 @@ Tdistfile_status Tdistfile::request(string msg) //Name the socket, as agreed with the server: address.sin_family = AF_INET; address.sin_addr.s_addr = inet_addr("127.0.0.1"); - address.sin_port = htons(9797); + address.sin_port = htons(9888); len = sizeof(address); //Connect your socket to the server’s socket: @@ -239,7 +239,7 @@ bool Tdistfile::choose_best_mirror(CURLM* cm, uint connection_num, uint network_ Pbest_mirror->start(); active_connections_num++; connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); - return 0; + return R_DOWNLOAD_STARTED; } else{ error_log("Can't choose mirror for segment:"+dn_segments[seg_num].file_name); @@ -272,7 +272,7 @@ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint ne network_array[network_num].benchmarked_mirror_list[best_mirror_num].start(); active_connections_num++; connection_array[connection_num].start(cm, network_num, num, &dn_segments[seg_num], best_mirror_num); - return 0; + return R_DOWNLOAD_STARTED; } else{ error_log("Can't choose LOCAL mirror for segment:"+dn_segments[seg_num].file_name); @@ -284,6 +284,73 @@ bool Tdistfile::choose_best_local_mirror(CURLM* cm, uint connection_num, uint ne } } +void Tdistfile::choose_networks_with_priority( + uint priority, + bool &allow_remote_mirrors, + int &best_local_network_num, + int &best_proxy_fetcher_network_num, + int &best_remote_network_num){ + for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ + //if network priority set then it's active + if (network_array[network_num].priority){ + if (network_array[network_num].priority==priority){ + debug(" network_priority="+toString(network_array[network_num].priority)); + switch (network_array[network_num].network_mode){ + case MODE_LOCAL:{ + if (network_array[network_num].has_free_connections()){ + if (network_distfile_brokers_array[network_num].some_mirrors_have_NOT_failed_yet()){ +// debug(" Allowed network#:"+toString(network_num)); + if ((best_local_network_num==-1) + or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_local_network_num=network_num; + debug(" Replace best LOCAL network to network#:"+toString(network_num)); + } + } + }else{ + if (network_array[network_num].only_local_when_possible){ + if (network_distfile_brokers_array[network_num].some_mirrors_have_NOT_failed_yet()){ + allow_remote_mirrors=false; + debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed"); + } + } + } + break; + } + case MODE_PROXY_FETCHER:{ + //replace this one by does_not_reject_connections +// if (network_array[network_num].has_free_connections()){ + if + ((best_proxy_fetcher_network_num==-1) +// or +// (network_array[best_proxy_fetcher_network_num].active_connections_num>network_array[network_num].active_connections_num) + ){ + best_proxy_fetcher_network_num=network_num; + debug(" Replace best_proxy_fetcher_network_num to network#:"+toString(network_num)); + debug(" Replace best_proxy_fetcher_network_num to network#:"+toString(best_proxy_fetcher_network_num)); + } +// } + break; + } + case MODE_REMOTE:{ + if (network_array[network_num].has_free_connections()){ + if + ((best_remote_network_num==-1) + or + (network_array[best_remote_network_num].active_connections_num>network_array[network_num].active_connections_num)){ + best_remote_network_num=network_num; + debug(" Replace best_remote_network_num to network#:"+toString(network_num)); + } + } + break; + } + } + //work with network + } + } + } +} + + int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ try{ for (uint cur_network_priority=10; cur_network_priority>0; cur_network_priority--){ @@ -300,73 +367,12 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ int best_proxy_fetcher_network_num=-1; int best_remote_network_num=-1; bool allow_remote_mirrors=true; - for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ - //if network priority set then it's active - if (network_array[network_num].priority){ - if (network_array[network_num].priority==cur_network_priority){ - debug(" network_priority="+toString(network_array[network_num].priority)); - switch (network_array[network_num].network_mode){ - case MODE_LOCAL:{ - if (network_array[network_num].has_free_connections()){ - if (network_distfile_brokers_array[network_num].some_mirrors_have_NOT_failed_yet()){ -// debug(" Allowed network#:"+toString(network_num)); - if ((best_local_network_num==-1) - or (network_array[best_local_network_num].active_connections_num>network_array[network_num].active_connections_num)){ - best_local_network_num=network_num; - debug(" Replace best LOCAL network to network#:"+toString(network_num)); - } - } - }else{ - if (network_array[network_num].only_local_when_possible){ - if (network_distfile_brokers_array[network_num].some_mirrors_have_NOT_failed_yet()){ - allow_remote_mirrors=false; - debug("Network"+toString(network_num)+" forbids using remote mirrors because not all local mirrors have failed"); - } - } - } - break; - } - case MODE_PROXY_FETCHER:{ - //replace this one by does_not_reject_connections -// if (network_array[network_num].has_free_connections()){ - if - ((best_proxy_fetcher_network_num==-1) -// or -// (network_array[best_proxy_fetcher_network_num].active_connections_num>network_array[network_num].active_connections_num) - ){ - best_proxy_fetcher_network_num=network_num; - debug(" Replace best_proxy_fetcher_network_num to network#:"+toString(network_num)); - debug(" Replace best_proxy_fetcher_network_num to network#:"+toString(best_proxy_fetcher_network_num)); - } -// } - break; - } - case MODE_REMOTE:{ - if (network_array[network_num].has_free_connections()){ - if - ((best_remote_network_num==-1) - or - (network_array[best_remote_network_num].active_connections_num>network_array[network_num].active_connections_num)){ - best_remote_network_num=network_num; - debug(" Replace best_remote_network_num to network#:"+toString(network_num)); - } - } - break; - } - } - //work with network - } - } - } - if (best_local_network_num!=-1){ - //best network has been found - //work with network - debug(" So best LOCAL network is network#:"+toString(best_local_network_num)); - return choose_best_local_mirror(cm, connection_num, best_local_network_num, seg_num); - }else{ -//DDOWNLOADED) return false; -//DPROXY_QUEUED) || (status==DPROXY_DOWNLOADING)) && (time_left<100)) return false; - //oterwise allow connections + choose_networks_with_priority(cur_network_priority, allow_remote_mirrors, best_local_network_num,best_proxy_fetcher_network_num,best_remote_network_num); + +//DDOWNLOADED +//DPROXY_QUEUED +//DPROXY_DOWNLOADING +//oterwise allow connections // DNEW, // D_NOT_PROXY_REQUESTED, // DPROXY_REJECTED, @@ -375,25 +381,35 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ // DDOWNLOADING, // DDOWNLOADED, // DFAILED + + if (best_local_network_num!=-1){ + //best network has been found + //work with network + debug(" So best LOCAL network is network#:"+toString(best_local_network_num)); + return choose_best_local_mirror(cm, connection_num, best_local_network_num, seg_num); + }else{ if (allow_remote_mirrors){ //since all local failed, go to proxy_fetcher debug("Remote mirrors are allowed"); if (best_proxy_fetcher_network_num != -1){ if (status == DPROXY_QUEUED){ - return 1; - }else{ - // request from proxy fethcer - status=request(json_data); - debug("Trying to dowload distfile"+name+" via proxy_fetcher. status="+toString(status)); - if (status==DPROXY_DOWNLOADED){ - // start download from the proxy_fetcher - debug(" So best proxy_fetcher_network is network#:"+toString(best_proxy_fetcher_network_num)); +// TO-DO: Add option to go for remote networks or low priority networks even if already DPROXY_QUEUED +// TO-DO: There can be several proxy-fetchers of the same priority level, define a rule to make a choice +// TO-DO: request, probably it's already DPROXY_DOWNLOADED + // check time from last request if small difference return +// return 1; + } + // request from proxy fethcer + status=request(json_data); + debug("Trying to dowload distfile"+name+" via proxy_fetcher. status="+toString(status)); + if (status==DPROXY_DOWNLOADED){ + // start download from the proxy_fetcher + debug(" So best proxy_fetcher_network is network#:"+toString(best_proxy_fetcher_network_num)); // BEWARE -- CORRECT FOLLOWING LINES !!!!!!!!!!! - // start download from proxy_fether mirrors +// TO-DO: start download from proxy_fether mirrors // return choose_best_mirror(cm, connection_num, best_remote_network_num, seg_num); - } - //return - don't switch to low priority networks - return 0; } + //return - don't switch to low priority networks + return R_ADDED_TO_PROXY_QUEUE; }else{ // remote_mirrors_go_third if (best_remote_network_num!=-1){ @@ -407,12 +423,12 @@ int Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){ }else{ debug("NOT all local mirrors have failed - restricted to local mirrors only."); //return - don't switch to low priority networks - return 100; + return R_WAIT_FOR_LOCAL_MIRRORS; } } } // haven't found anything suitable - return 120; + return R_NO_FREE_NETWORK_CONNECTION_FOUND; }catch(...){ error_log("Error: distfile.cpp: provide_segment()"); return 1; diff --git a/segget/distfile.h b/segget/distfile.h index 1b2f110..abf8076 100644 --- a/segget/distfile.h +++ b/segget/distfile.h @@ -71,6 +71,11 @@ using namespace std; typedef unsigned int uint; +#define R_DOWNLOAD_STARTED 0 +#define R_ADDED_TO_PROXY_QUEUE 100 +#define R_WAIT_FOR_LOCAL_MIRRORS 101 +#define R_NO_FREE_NETWORK_CONNECTION_FOUND 102 + enum Tdistfile_status{ DNEW, D_NOT_PROXY_REQUESTED, @@ -138,6 +143,7 @@ class Tdistfile{ void load_distfile_from_json(json_object* json_obj_distfile); void load_url_list(json_object* json_array_distfile_urllist); void split_into_segments(); + void choose_networks_with_priority(uint priority, bool &allow_remote_mirrors, int &best_local_network_num,int &best_proxy_fetcher_network_num,int &best_remote_network_num); int provide_segment(CURLM* cm, uint connection_num, uint seg_num); void inc_dld_segments_count(Tsegment * current_segment); void symlink_distfile_to_provide_mirror_dir(); diff --git a/segget/network0.conf b/segget/network0.conf index 6b80614..6eee6da 100644 --- a/segget/network0.conf +++ b/segget/network0.conf @@ -63,7 +63,7 @@ bind_interface=none # Maximum value: 20 # Default: # max_connections=10 -max_connections=20 +max_connections=1 # CONNECTION_TIMEOUT # Set the number of seconds to wait while trying to connect. Use 0 to wait diff --git a/segget/pkg.cpp b/segget/pkg.cpp index 84dc9f6..bcbac68 100644 --- a/segget/pkg.cpp +++ b/segget/pkg.cpp @@ -26,18 +26,47 @@ #include "pkg.h" +void Tpkg::push_back_distfile(json_object* json_distfile){ + try{ + distfile_count++; + Tdistfile * Pcur_distfile=new Tdistfile(); + Pdistfile_list.push_back(Pcur_distfile); + Pcur_distfile->load_distfile_from_json(json_distfile); + }catch(...){ + error_log("Error in pkg.cpp: pushback_distfile()"); + } +} + + +void Tpkg::load_distfile_list(json_object* json_array_distfile_list){ + try{ + ulong distfile_counter=json_object_array_length(json_array_distfile_list); + for(uint distfile_array_item_num=0;distfile_array_item_num<distfile_counter;distfile_array_item_num++){ + push_back_distfile(json_object_array_get_idx(json_array_distfile_list,distfile_array_item_num)); + } + }catch(...){ + error_log("Error in pkg.cpp: load_distfile_list()"); + } +} +/* void Tpkg::load_distfile_list(json_object* json_array_distfile_list){ try{ distfile_count=json_object_array_length(json_array_distfile_list); - Pdistfile_list= new Tdistfile* [distfile_count]; +// Pdistfile_list= new Tdistfile* [distfile_count]; for(uint distfile_array_item_num=0;distfile_array_item_num<distfile_count;distfile_array_item_num++){ - Pdistfile_list[distfile_array_item_num]=new Tdistfile(); +// Pdistfile_list[distfile_array_item_num]=new Tdistfile(); + Pdistfile_list.push_back(new Tdistfile()); +// Tdistfile a_distfile; +// a_distfile.load_distfile_from_json(json_object_array_get_idx(json_array_distfile_list,distfile_array_item_num)); Pdistfile_list[distfile_array_item_num]->load_distfile_from_json(json_object_array_get_idx(json_array_distfile_list,distfile_array_item_num)); } }catch(...){ error_log("Error in pkg.cpp: load_distfile_list()"); } } +*/ + + void Tpkg::load_pkg_from_json(json_object* json_obj_pkg){ try{ @@ -54,7 +83,8 @@ Tpkg::~Tpkg(){ try{ for (uint i=0;i<distfile_count;i++) delete Pdistfile_list[i]; - delete [] Pdistfile_list; + Pdistfile_list.clear(); +// delete [] Pdistfile_list; }catch(...){ error_log("Error in pkg.cpp: ~Tpkg"); } diff --git a/segget/pkg.h b/segget/pkg.h index 660dde2..480c24f 100644 --- a/segget/pkg.h +++ b/segget/pkg.h @@ -35,18 +35,29 @@ using namespace std; typedef unsigned int uint; +typedef Tdistfile* Pdistfile; class Tpkg{ public: - Tdistfile **Pdistfile_list; + vector<Pdistfile> Pdistfile_list; string name; string category; uint distfile_count; void load_distfile_list(json_object* json_array_distfile_list); - Tpkg(): Pdistfile_list(0),name(""),category(""), distfile_count(0){}; - Tpkg(const Tpkg &L); // copy constructor + Tpkg(): + Pdistfile_list(), + name(""), + category(""), + distfile_count(0) + {}; + Tpkg(const Tpkg &L); // copy constructor Tpkg & operator=(const Tpkg &L); ~Tpkg(); + void push_back_distfile(json_object* json_distfile); void load_pkg_from_json(json_object* json_obj_pkg); }; + +Tpkg **Ppkg_array; +Tpkg proxy_fetcher_pkg; + #endif
\ No newline at end of file diff --git a/segget/proxyfetcher.cpp b/segget/proxyfetcher.cpp new file mode 100644 index 0000000..67ab164 --- /dev/null +++ b/segget/proxyfetcher.cpp @@ -0,0 +1,105 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "proxyfetcher.h" + +void *run_proxy_fetcher_server(void * ){ + int server_sockfd, client_sockfd; + socklen_t server_len, client_len; + struct sockaddr_in server_address; + struct sockaddr_in client_address; + int result; + fd_set readfds, testfds; + + // Create and name a socket for the server: + server_sockfd = socket(AF_INET, SOCK_STREAM, 0); + + server_address.sin_family = AF_INET; + //server_address.sin_addr.s_addr = htonl(INADDR_ANY); + server_address.sin_addr.s_addr = inet_addr(settings.provide_proxy_fetcher_ip.c_str()); + server_address.sin_port = htons(settings.provide_proxy_fetcher_port); + server_len = sizeof(server_address); + + bind(server_sockfd, (struct sockaddr *)&server_address, server_len); + + //Create a connection queue and initialize readfds to handle input from server_sockfd: + listen(server_sockfd, 5); + + FD_ZERO(&readfds); + FD_SET(server_sockfd, &readfds); + + //Now wait for clients and requests. Because you have passed a null pointer as the timeout parameter, no timeout will occur. The program will exit and report an error if select returns a value less than 1: + while(1) { + int fd; + int nread; + testfds = readfds; + + debug("proxyfether is waiting for connections"); + result = select(FD_SETSIZE, &testfds, (fd_set *)0, (fd_set *)0, (struct timeval *) 0); +// debug("proxyfether done waiting"); + + if(result < 1) { + error_log("Error in proxyfetcher.cpp : run"); + exit(1); + } + + //Once you know you’ve got activity, you can find which descriptor it’s on by checking each in turn using FD_ISSET: + for(fd = 0; fd < FD_SETSIZE; fd++) { + if(FD_ISSET(fd,&testfds)) { + + //If the activity is on server_sockfd, it must be a request for a new connection, and you add the associated client_sockfd to the descriptor set: + + if(fd == server_sockfd) { + debug("new client - read"); + client_len = sizeof(client_address); + client_sockfd = accept(server_sockfd, + (struct sockaddr *)&client_address, &client_len); + FD_SET(client_sockfd, &readfds); + debug("adding client on fd:"+toString(client_sockfd)); + + //If it isn’t the server, it must be client activity. If close is received, the client has gone away, and you remove it from the descriptor set. Otherwise, you “serve” the client as in the previous examples. + }else{ + debug("old client - read"); + ioctl(fd, FIONREAD, &nread); + if(nread == 0) { + close(fd); + FD_CLR(fd, &readfds); + debug("removing client on fd:"+toString(fd)); + }else{ + char buffer[100000]=""; + read(fd, &buffer, nread); + debug("serving client - read"); + debug("serving client on fd"+toString(fd)); + string recv_msg=buffer; + error_log("Received a msg from the client:"+recv_msg); + + proxy_fetcher_pkg.push_back_distfile(json_tokener_parse(buffer)); + } + } + } + } + } +} diff --git a/segget/proxyfetcher.h b/segget/proxyfetcher.h new file mode 100644 index 0000000..4300f06 --- /dev/null +++ b/segget/proxyfetcher.h @@ -0,0 +1,45 @@ +/* +* Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn <fastinetserver@gmail.com>. +* +* Project: IDFetch. +* Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). +* Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). +* Mentoring organization: Gentoo Linux. +* Sponsored by GSOC 2010. +* +* This file is part of Segget. +* +* Segget is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* Segget is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with Segget; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef __PROXYFETCHER_H__ +#define __PROXYFETCHER_H__ + +#include <sys/types.h> +#include <sys/socket.h> +#include <stdio.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <sys/time.h> +#include <sys/ioctl.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <json/json.h> +#include "tui.h" +#include "pkg.h" + +void *run_proxy_fetcher_server(void * ); +#endif
\ No newline at end of file diff --git a/segget/segget.conf b/segget/segget.conf index 489acf9..5e103c0 100644 --- a/segget/segget.conf +++ b/segget/segget.conf @@ -76,7 +76,7 @@ max_tries=10 # Maximum value: 20 # Default: # max_connections=10 -max_connections=10 +max_connections=3 # CURRENT_SPEED_TIME_INTERVAL_MSECS # segget transfers may have bursty nature of their traffic. Therefore, while @@ -174,6 +174,26 @@ provide_mirror_dir=./provide_mirror_dir # provide_mirror_files_restrict_list_on=0 provide_mirror_files_restrict_list_on=1 +[provide_proxy_fetcher_to_others] +# tuiclient monitors segget's activity by establishing tcp connection +# with segget daemon (ui_server part of it). + +# PROVIDE_PROXY_FETCHER_IP +# Define an ip address segget will use to provide access for tuiclients. +# The parameter should be a string holding your host dotted IP address. +# Default: +# provide_proxy_fetcher_ip=127.0.0.1 +provide_proxy_fetcher_ip=127.0.0.1 + +# PROVIDE_PROXY_FETCHER_PORT +# Define a port segget will use to provide access for tuiclients. +# The parameter should be an integer. +# Minimum value: 1 +# Maximum value: 65535 +# Default: +# provide_proxy_fetcher_port=9777 +provide_proxy_fetcher_port=9777 + [networks] # NETWORK0_PRIORITY # Define priority as a value in range from lowest 0 to highest 10. @@ -188,8 +208,17 @@ provide_mirror_files_restrict_list_on=1 # instead of # should be network number. For network0 it's network0.conf # Default: # network0_priority=10 +# network1_priority=0 +# network2_priority=0 +# network3_priority=0 +# network4_priority=0 +# network5_priority=0 +# network6_priority=0 +# network7_priority=0 +# network8_priority=0 +# network9_priority=0 network0_priority=10 -network1_priority=0 +network1_priority=9 network2_priority=8 network3_priority=0 network4_priority=0 @@ -213,6 +242,26 @@ user_agent=segget # NOT IMPLEMENTED YET: ftp-user=anonymous # NOT IMPLEMENTED YET: ftp-password=me@mail.ru +[ui_server] +# tuiclient monitors segget's activity by establishing tcp connection +# with segget daemon (ui_server part of it). + +# UI_IP +# Define an ip address segget will use to provide access for tuiclients. +# The parameter should be a string holding your host dotted IP address. +# Default: +# ui_ip=127.0.0.1 +ui_ip=127.0.0.1 + +# UI_PORT +# Define a port segget will use to provide access for tuiclients. +# The parameter should be an integer. +# Minimum value: 1 +# Maximum value: 65535 +# Default: +# ui_port=9999 +ui_port=9999 + [logs] # LOGS_DIR # Define a dir to store log files. diff --git a/segget/segget.cpp b/segget/segget.cpp index dc2692c..8aa11ce 100644 --- a/segget/segget.cpp +++ b/segget/segget.cpp @@ -64,6 +64,7 @@ int load_pkgs(){ else { stats.pkg_count=json_object_array_length(json_array_pkg_list); Ppkg_array= new Tpkg* [stats.pkg_count]; + // create 0 pkg for distfiles to provide proxy-fetcher for(uint array_item_num=0;array_item_num<stats.pkg_count;array_item_num++){ Ppkg_array[array_item_num]=new Tpkg; Ppkg_array[array_item_num]->load_pkg_from_json(json_object_array_get_idx(json_array_pkg_list,array_item_num)); @@ -79,6 +80,7 @@ int load_pkgs(){ return 1; } } +/* void show_pkgs(){ try{ for (uint array_item_num=0;array_item_num<stats.pkg_count;array_item_num++){ @@ -94,31 +96,29 @@ void show_pkgs(){ error_log("Error in segget.cpp: show_pkgs()"); } } - -int choose_segment(uint connection_num){ - try{ - uint pkg_num (0); +*/ +int pkg_choose_segment(Tpkg * cur_pkg, uint connection_num){ uint distfile_num(0); uint segment_num(0); - while (pkg_num<stats.pkg_count){ -// debug("pkg_num:"+toString(pkg_num)); - while(distfile_num<Ppkg_array[pkg_num]->distfile_count){ - if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->allows_new_actions()){ - debug("Distfile "+Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->name+" allows new connections"); + while(distfile_num<cur_pkg->distfile_count){ +// if (Ppkg_array[pkg_num]->distfile_vector[distfile_num].allows_new_actions()){ + if (cur_pkg->Pdistfile_list[distfile_num]->allows_new_actions()){ + debug("Distfile "+cur_pkg->Pdistfile_list[distfile_num]->name+" allows new connections"); +// debug("Distfile "+Ppkg_array[pkg_num]->distfile_vector[distfile_num]->name+" allows new connections"); // debug(" distfile_num:"+toString(distfile_num)); - if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->active_connections_num<settings.max_connection_num_per_distfile){ + if (cur_pkg->Pdistfile_list[distfile_num]->active_connections_num<settings.max_connection_num_per_distfile){ debug("max_connection limit has not been reached"); debug("segment_num:"+toString(segment_num)); - debug("segment_count:"+toString(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->segments_count)); - while (segment_num<Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->segments_count){ + debug("segment_count:"+toString(cur_pkg->Pdistfile_list[distfile_num]->segments_count)); + while (segment_num<cur_pkg->Pdistfile_list[distfile_num]->segments_count){ debug("segment_num:"+toString(segment_num)); // debug(" segment_num:"+toString(segment_num)); // segments_in_progress[connection_num]= // if not(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->get_segment_downloaded_status(segment_num); - debug("Let's get segment status"+statusToString(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->dn_segments[segment_num].status)); - if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->dn_segments[segment_num].status==SWAITING){ - if ( ! Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num)){ - return 0; // success + debug("Let's get segment status"+statusToString(cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status)); + if (cur_pkg->Pdistfile_list[distfile_num]->dn_segments[segment_num].status==SWAITING){ + if ( ! cur_pkg->Pdistfile_list[distfile_num]->provide_segment(cm, connection_num, segment_num)){ + return 0; // download started }; }else{ debug("status is not SWAITING - go for the next segment"); @@ -127,8 +127,8 @@ int choose_segment(uint connection_num){ segment_num++; } }else{ - debug(" distfile "+Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->name+" has " - +toString(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->active_connections_num) + debug(" distfile "+cur_pkg->Pdistfile_list[distfile_num]->name+" has " + +toString(cur_pkg->Pdistfile_list[distfile_num]->active_connections_num) +" connections => choosing another distfile."); } } @@ -136,7 +136,20 @@ int choose_segment(uint connection_num){ segment_num=0; } distfile_num=0; - pkg_num++; + return 1; +} + +int choose_segment(uint connection_num){ + try{ + for (uint pkg_num=0; pkg_num<stats.pkg_count; pkg_num++){ +// debug("pkg_num:"+toString(pkg_num)); + if (! pkg_choose_segment(Ppkg_array[pkg_num], connection_num)){ + return 0; + } + } + // download distfiles as a proxy-fetcher + if (! pkg_choose_segment(&proxy_fetcher_pkg, connection_num)){ + return 0; } // for (uint array_item_num=0;array_item_num<pkg_count;array_item_num++){ //for(int distfile_array_item_num=0;distfile_array_item_num<Ppkg_array[array_item_num]->distfile_count;distfile_array_item_num++){ @@ -278,7 +291,16 @@ void launch_ui_server_thread(){ debug_no_msg("Creating ui_server_thread."); ui_server.init(); iret1 = pthread_create( &ui_server_thread, NULL, run_ui_server, (void*) NULL); - debug_no_msg("ui_server_lanched"); + debug_no_msg("ui_server_thread launched"); +} + +void launch_proxy_fetcher_server_thread(){ + pthread_t proxy_fetcher_server_thread; + int iret1; + debug_no_msg("Creating ui_server_thread."); +// proxy_fetcher_server_thread.init(); + iret1 = pthread_create( &proxy_fetcher_server_thread, NULL, run_proxy_fetcher_server, (void*) NULL); + debug_no_msg("proxy_fetcher_server_thread launched"); } void segget_exit(int sig){ @@ -334,6 +356,11 @@ int main() error_log_no_msg("Error in segget.cpp launch_ui_server() failed"); } try{ + launch_proxy_fetcher_server_thread(); + }catch(...){ + error_log_no_msg("Error in segget.cpp launch_proxy_fetcher_server_thread failed"); + } + try{ launch_tui_thread(); }catch(...){ error_log_no_msg("Error in segget.cpp launch_tui_theread() failed"); diff --git a/segget/segget.h b/segget/segget.h index f224609..d2e9b2c 100644 --- a/segget/segget.h +++ b/segget/segget.h @@ -50,15 +50,15 @@ #include "tui.cpp" #include "utils.cpp" #include "ui_server.cpp" +#include "proxyfetcher.cpp" using namespace std; -Tpkg **Ppkg_array; - CURLM *cm; int load_pkgs(); void show_pkgs(); +int pkg_choose_segment(Tpkg * cur_pkg, uint connection_num); int choose_segment(uint connection_num); int download_pkgs(); int main(); diff --git a/segget/settings.cpp b/segget/settings.cpp index b60e2d0..ddab8b9 100644 --- a/segget/settings.cpp +++ b/segget/settings.cpp @@ -95,6 +95,9 @@ void Tsettings::init(){ load_provide_mirror_files_restricted_patterns_vector(); } + conf.set(provide_proxy_fetcher_ip, "provide_proxy_fetcher_to_others", "provide_proxy_fetcher_ip"); + conf.set(provide_proxy_fetcher_port, "provide_proxy_fetcher_to_others", "provide_proxy_fetcher_port",1,65535); + ulong cur_network_priority; for (uint network_num=0; network_num<MAX_NETWORKS; network_num++){ //set default values, in case segget.conf doesn't have these settings @@ -109,6 +112,10 @@ void Tsettings::init(){ network_array[network_num].init(cur_network_priority); } } + + conf.set(ui_ip, "ui_server", "ui_ip"); + conf.set(ui_port, "ui_server", "ui_port",1,65535); + conf.clear(); }catch(...){ error_log_no_msg("Error in settings.cpp: init()"); diff --git a/segget/settings.h b/segget/settings.h index c746888..ede0a7a 100644 --- a/segget/settings.h +++ b/segget/settings.h @@ -68,6 +68,12 @@ class Tsettings{ string provide_mirror_dir; bool provide_mirror_files_restrict_list_on; vector<string> provide_mirror_files_restricted_patterns_vector; + //provide_proxy_fether_to_others + string provide_proxy_fetcher_ip; + ulong provide_proxy_fetcher_port; + //ui_server + string ui_ip; + ulong ui_port; //logs string logs_dir; string general_log_file; @@ -104,6 +110,12 @@ class Tsettings{ provide_mirror_dir("none"), provide_mirror_files_restrict_list_on(0), provide_mirror_files_restricted_patterns_vector(), + //provide_proxy_fether_to_others + provide_proxy_fetcher_ip("127.0.0.1"), + provide_proxy_fetcher_port(9888), + //ui_server + ui_ip("127.0.0.1"), + ui_port(9999), //logs logs_dir("./logs"), general_log_file("segget.log"), diff --git a/segget/ui_server.cpp b/segget/ui_server.cpp index 6cb000f..2ba19d9 100644 --- a/segget/ui_server.cpp +++ b/segget/ui_server.cpp @@ -42,9 +42,9 @@ void Tui_server::init(){ server_address.sin_family = AF_INET; //server_address.sin_addr.s_addr = htonl(INADDR_ANY); - string bind_address="127.0.0.1"; + string bind_address=settings.ui_ip; server_address.sin_addr.s_addr = inet_addr(bind_address.c_str()); - server_address.sin_port = htons(9999); + server_address.sin_port = htons(settings.ui_port); server_len = sizeof(server_address); int res; res=bind(server_sockfd, (struct sockaddr *)&server_address, server_len); |