connect(); # For page performance metrics $now = $Mirror->microtime_float(); # Log download stats, and to which table. log_download_table will be suffixed with a number later $log_download = !$App->getDBReadOnly(); $log_download_table = "downloads"; $_debug = false; # Append /debug6687 to file for debugging output # wget -S -O - download.eclipse.org/technology/epp/downloads/release/neon/3/eclipse-modeling-neon-3-win32.zip/debug6687 # Files under this size get served-up without mirrors. # handled by Apache RewriteCond $smallfile_threshold = 2000000; # Do a reverse lookup on the client IP - for internal mirror matching and logging $remote_addr = $_SERVER['REMOTE_ADDR']; $ccode = $Mirror->selectCountryCodeByIP($remote_addr); # use a proxy? $proxy = ""; if ($_SERVER['SERVER_NAME'] == "download.eclipse.org") { $proxy = "http://proxy.eclipse.org:9898"; } # project and file index information for logging and statistics $uri = parse_url($_SERVER['REQUEST_URI']); $_file = $uri['path']; # Enable debug if(preg_match('/debug6687$/', $_file)) { $_file = str_replace("/debug6687", "", $_file); $_debug = true; header("Content-type: text/html"); } header("X-IP-Geolocation: " . $ccode); header("X-File: " . $_file); $file_id = 0; $client_hostname = @gethostbyaddr($remote_addr); $filetime = 0; $filesize = 0; $filetime_update = false; if($_file == "") { echo "Invalid file"; return false; } # locate file on download.eclipse.org to ensure it's valid $filename = $_SERVER['DOCUMENT_ROOT'] . $_file; if(@fopen($filename, "r")) { $filetime = @filemtime($filename); $filesize = @filesize($filename); } else { header("HTTP/1.0 404 Not Found"); cleanUpAndExit(); } # Before doing anything else, is this file cached? if(handleIfModified($filename, $filetime)) { cleanUpAndExit(); } # Bypass the entire thing if the file is in the excluded list if($Mirror->isExcluded($_file)) { header("X-Excluded-File: true"); # Don't log this download, since listing the file in the download_index will # cause the mirror trends to be off closeDB(); serveFile($filename); cleanUpAndExit(); } # q1: get file info from slave $sql = "SELECT IDX.file_id, IDX.timestamp_disk, IDX.size_disk_bytes, IF(IDX.md5sum = '0', '', IDX.md5sum) AS md5sum, IF(IDX.sha1sum = '0', '', IDX.sha1sum) AS sha1sum, IF(IDX.sha512sum = '0', '', IDX.sha512sum) AS sha512sum FROM download_file_index AS IDX WHERE IDX.file_name = '$_file'"; $rs = mysql_query($sql, $dbh_RW); if($myrow = mysql_fetch_assoc($rs)) { $file_id = $myrow['file_id']; if($filetime != $myrow['timestamp_disk']) { $filetime_update = true; } $md5sum = $myrow['md5sum']; $sha1sum = $myrow['sha1sum']; $sha512sum = $myrow['sha512sum']; } else { # Add to index $file_id = &addFileToIndex($_file); } if($filetime_update) { updateFileIndex($_file_id); } $sql = "SELECT key_value FROM SYS_variables WHERE key_name = 'download_table' LIMIT 1"; $rs = mysql_query($sql, $dbh_RW); if($myrow = mysql_fetch_assoc($rs)) { $key_value = $myrow['key_value']; } $log_download_table .= $key_value; if($_debug) { header("X-File-ID: " . $file_id); } # We have a file. Should we just serve it, or redirect to a mirror # if($filesize < $smallfile_threshold || substr($remote_addr, 0, 10) == "198.41.30." || !$Mirror->isValidPublicIP($remote_addr)) { if(substr($remote_addr, 0, 10) == "198.41.30." || !$Mirror->isValidPublicIP($remote_addr)) { header("X-IP-Bypass: true"); logDownload($file_id, 1, $remote_addr); closeDB(); serveFile($filename); cleanUpAndExit(); } else { # Pull mirror $sql = ""; # Bug 520252 - Erroneous mirror: make sure mirror's last_known_sync is 3 hours ago to allow more sync time. # Bug 517294 - increase to 6 hrs (21600 s) $having = "((MIR.is_internal <> 1 AND (last_known_sync + 21600) > " . $filetime . ") OR LOCATE(MIR.internal_host_pattern, '" . $client_hostname . "') > 1) AND success_rate > 99 "; $where = " MRD.drop_id = 'EclipseFull'"; $sql = "SELECT /* download.php:$remote_addr */ IF(LOCATE(MIR.internal_host_pattern, '" . $client_hostname . "') > 1, 1, 0) AS SuggestInternal, IF(COU.ccode = '$ccode', 1, 0) AS SameCountry, IF(COU.continent_code = MYCOU.continent_code, 1, 0) AS SameContinent, ROUND(AVG(TRD.download_speed_kbps),0) * RAND() AS avg_speed_indexed, SUM(IF(TRD.http_status_code BETWEEN 200 AND 299, 1, 0)) / COUNT(TRD.trend_id) * 100 AS success_rate, MIR.mirror_id, MIR.organization, MIR.ccode, MIR.is_internal, MIR.internal_host_pattern, MRD.drop_id, DRP.our_path, DRP.file_pattern, MRP.protocol, MRP.base_path, MRD.rel_path_override, MRD.timestamp AS last_known_sync, COU.en_description AS country_desc, CON.en_description AS continent_desc, CON.sort_order, DFM.last_http_code FROM mirrors AS MIR INNER JOIN mirror_protocols AS MRP on MRP.mirror_id = MIR.mirror_id INNER JOIN mirror_drops AS MRD ON MRD.mirror_id = MIR.mirror_id AND MRD.protocol = MRP.protocol INNER JOIN drops as DRP ON DRP.drop_id = 'EclipseFull' INNER JOIN SYS_countries AS COU ON COU.ccode = MIR.ccode INNER JOIN SYS_continents AS CON ON CON.continent_code = COU.continent_code LEFT JOIN SYS_countries AS MYCOU ON MYCOU.ccode = '$ccode' LEFT JOIN mirror_trends AS TRD ON TRD.mirror_id = MIR.mirror_id LEFT JOIN download_file_mirrors AS DFM ON (DFM.file_id = $file_id AND DFM.mirror_id = MIR.mirror_id) WHERE MIR.is_advertise = 1 AND MIR.create_status = 'active' AND $where GROUP BY MIR.mirror_id HAVING " . $having . " ORDER BY success_rate DESC, SameCountry DESC, SameContinent DESC"; # The order is important, as we only pick the first record! $querytime = $Mirror->microtime_float() - $now; $rs = mysql_query($sql, $dbh_RW); if($myrow = mysql_fetch_assoc($rs)) { $_url = $myrow['base_path']. $_file; header("X-Mirror-ID: " . $myrow['mirror_id']); header("X-File-ID: " . $file_id); header("X-Mirror-Success: " . $myrow['success_rate']); header("X-Pagetime: " . $querytime); if($_debug) { echo $sql; cleanUpAndExit(); } # Bug 517294 - update site goes to a mirror site although not mirrored if($myrow['last_http_code'] == "200") { logDownload($file_id, $myrow['mirror_id'], $remote_addr); redirectToMirror($_url, $filename, $filetime); } else { $httpCode = $myrow['last_http_code']; # Avoid banging on the same mirror all the time, unless there is no code in the db if($httpCode == "") { # This mirror does not have this file. Fetch mirror headers to see if it works $curl = curl_init($_url); curl_setopt($curl, CURLOPT_URL, $_url); curl_setopt($curl, CURLOPT_FILETIME, true); curl_setopt($curl, CURLOPT_NOBODY, true); # Perform HEAD request curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); if($proxy != "") { curl_setopt($curl, CURLOPT_PROXY, $proxy); } curl_setopt($curl, CURLOPT_HEADER, true); $response = curl_exec($curl); $httpCode = curl_getinfo($curl, CURLINFO_HTTP_CODE); if(!is_numeric($httpCode)) { $httpCode = 0; } curl_close($curl); $sql = "INSERT INTO download_file_mirrors SET file_id = " . $file_id . ", mirror_id = " . $myrow['mirror_id'] . ", last_http_code = '" . $httpCode . "', last_checked = NOW() ON DUPLICATE KEY UPDATE last_http_code = '" . $httpCode . "', last_checked = NOW()"; mysql_query($sql, $dbh_RW); } if($httpCode == "200") { header("X-Mirror-Override: mirror-updated-200"); logDownload($file_id, $myrow['mirror_id'], $remote_addr); redirectToMirror($_url, $filename, $filetime); } else { logDownload($file_id, 1, $remote_addr); header("X-Mirror-Override: mirror-not-updated"); closeDB(); serveFile($filename); } } } else { logDownload($file_id, 1, $remote_addr); closeDB(); header("X-Mirror-ID: nomirror"); header("X-Pagetime: " . $querytime); if($_debug) { echo $sql; cleanUpAndExit(); } serveFile($filename); } } cleanUpAndExit(); /** * Send the actual file to the browser * @param String $full_filename * @since 2015-06-23 * @author droy */ function serveFile($full_filename) { global $filesize, $filetime, $_debug; header('Content-Description: File Transfer'); # header('Content-Type: ' . finfo_file($finfo, $full_filename)); # If we had pecl header('Content-Type: application/octet-stream'); header('Content-Disposition: attachment; filename=' . basename($full_filename)); header('Content-Transfer-Encoding: binary'); header('Last-Modified: ' . date("r", $filetime)); # Thu, 18 Jun 2015 20:20:37 GMT header('ETag: ' . calculateEtag($full_filename, date("r", $filetime))); header('Pragma: public'); header('Content-Length: ' . $filesize); ob_clean(); ob_end_flush(); if(!$_debug) { readfile($full_filename); } } /** * Log the download to the downloads table * @param int $in_file_id * @param int $in_mirror_id * @param string $ip * @since 2015-06-23 * @author droy */ function logDownload($in_file_id, $in_mirror_id, $ip = NULL) { if (is_null($ip)) { $ip = $_SERVER['REMOTE_ADDR']; } global $dbh_RW, $log_download_table, $client_hostname, $ccode, $_debug; if ($in_file_id != "") { $sql = "INSERT DELAYED INTO $log_download_table (file_id, download_date, remote_host, remote_addr, mirror_id, ccode) VALUES ( $in_file_id, NOW(), '$client_hostname', '$ip', $in_mirror_id, '$ccode') "; mysql_query($sql, $dbh_RW); $sql = "UPDATE LOW_PRIORITY download_file_index SET download_count = download_count + 1 WHERE file_id = " . $in_file_id; mysql_query($sql, $dbh_RW); } } /** * Add file to the download index * @param int $in_file * @return int file_id of the added file * @since 2015-06-23 * @author droy */ function addFileToIndex($in_file) { global $App, $dbc_RW, $dbh_RW, $filetime, $filesize, $_debug; # Add this file to the file index if($in_file != "") { $sql = "INSERT INTO download_file_index (file_id, file_name, download_count, timestamp_disk, size_disk_bytes) VALUES (NULL, '" . $App->sqlSanitize($in_file, $dbh_RW) . "', 0, $filetime, $filesize)"; $rs = mysql_query($sql, $dbh_RW); return mysql_insert_id($dbh_RW); } else { return 0; } } /** * Update download index * @param int $in_file * @return bool * @since 2015-10-05 * @author droy */ function updateFileIndex($in_file) { global $App, $dbc_RW, $dbh_RW, $filetime, $filesize, $_debug; $rValue = false; # Add this file to the file index if($in_file != "") { $sql = "UPDATE download_file_index SET timestamp_disk = " . $App->sqlSanitize($filetime, $dbh_RW) . ", size_disk_bytes = " . $App->sqlSanitize($filesize, $dbh_RW) . ", WHERE file_id = " . $App->sqlSanitize($in_file, $dbh_RW); $rValue = mysql_query($sql, $dbh_RW); } return $rValue; } /** * Calculate ETag based on absolute filename and last-modified date * @param string file_name * @param string last_modified date, in format Thu, 18 Jun 2015 20:20:37 GMT * @return string Etag value * @since 2017-09-29 * @author droy */ function calculateEtag($_filename, $_last_modified) { if($_filename != "" && $_last_modified != "") { return '"' . md5($_filename . $_last_modified) . '"'; } else { return false; } } /** * Calculate ETag based on absolute filename and last-modified date * @param string file_name * @param string last_modified date, in format Thu, 18 Jun 2015 20:20:37 GMT * @return bool is_304 * @since 2017-09-29 * @author droy */ function handleIfModified($_filename, $_last_modified) { $is_304 = false; if($_filename != "" && $_last_modified != "") { if ( (isset($_SERVER['HTTP_IF_MODIFIED_SINCE']) && strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']) >= $filetime) || (isset($_SERVER['HTTP_IF_NONE_MATCH']) && $_SERVER['HTTP_IF_NONE_MATCH'] == calculateEtag($_filename, date("r", $_last_modified))) ) { header('HTTP/1.1 304 Not Modified'); $is_304 = true; } } return $is_304; } /** * Redirect request to a mirror * @param string URL * @param string Optional filename for ETag * @param string Optional filetime for ETag * @return null * @since 2017-09-29 * @author droy */ function redirectToMirror($_url, $_filename=null, $_filetime=null) { header("HTTP/1.1 307 Temporary Redirect"); if(isset($_filename) && isset($_filetime)) { header('ETag: ' . calculateEtag($_filename, date("r", $_filetime))); } header("Location: " . $_url); } /** * Clean up, disconnect and exit * @since 2017-09-29 * @author droy */ function cleanUpAndExit() { closeDB(); exit; } /** * Close the database connection. We do this separately to allow the DB to be * closed to liberate sleeping connections while we are serving files * @since 2018-04-18 * @author droy */ function closeDB() { global $dbc_RW; $dbc_RW->disconnect(); }