PHP removeBOM函数代码示例

function translate_text($text, $from = 'en', $to = 'en')
{
    global $googleKey;
    try {
        $url = 'https://www.googleapis.com/language/translate/v2/detect?key=' . $googleKey . '&q=' . urlencode($text);
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
        $result = curl_exec($ch);
        curl_close($ch);
        $result = json_decode(removeBOM($result));
        $language = '';
        if (!empty($result->data->detections[0][0]->language)) {
            if ($result->data->detections[0][0]->confidence < 0.05) {
                return false;
            } else {
                $language = $result->data->detections[0][0]->language;
            }
        } else {
            return false;
        }
        if ($language == $to || empty($to) || empty($language)) {
            return false;
        }
        $url = 'https://www.googleapis.com/language/translate/v2?key=' . $googleKey . '&q=' . urlencode($text) . '&source=' . $language . '&target=' . $to;
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
        $result = curl_exec($ch);
        curl_close($ch);
        $result = json_decode(removeBOM($result));
        return $result->data->translations[0]->translatedText;
    } catch (Exception $e) {
        return false;
    }
}

 /**
  * Loads CSV-formatted data from a stream, parses it and returns an array of arrays.
  *
  * It closes the stream before returning.
  * Data should be comma-delimited and string may be enclosed on double quotes.
  *
  * @param resource          $handle    An opened stream.
  * @param array|string|null $columns   Either:<ul>
  *                                     <li> an array of column names,
  *                                     <li> a string of comma-delimited (and possibly quoted) column names,
  *                                     <li> null (or ommited) to read column names from the first row of data.
  *                                     </ul>
  * @param string            $delimiter The character used for separating fields. If not specified, it will be
  *                                     auto-decteded, if possible. If it's not possible, a comma will be used.
  * @return array The loaded data.
  */
 static function loadCsvFromStream($handle, $columns = null, $delimiter = '')
 {
     if (is_null($columns)) {
         $columns = removeBOM(fgets($handle));
     }
     if (is_string($columns)) {
         if (!$delimiter) {
             $delimiter = self::autoDetectSeparator($columns);
         }
         $columns = map(explode($delimiter, $columns), function ($col) {
             return preg_replace("/^[\\s\"\t']|[\\s\"\t']\$/", '', $col);
         });
     } else {
         if (!$delimiter) {
             $delimiter = ',';
         }
     }
     // use fgetcsv which tends to work better than str_getcsv in some cases
     $data = [];
     $i = 0;
     $row = '';
     try {
         while ($row = fgetcsv($handle, null, $delimiter, '"')) {
             ++$i;
             $data[] = array_combine($columns, $row);
         }
         fclose($handle);
     } catch (ErrorException $e) {
         echo "\nInvalid row #{$i}\n\nColumns:\n";
         var_export($columns);
         echo "\n\nRow:\n";
         var_export($row);
         echo "\n";
         exit(1);
     }
     return $data;
 }

function translate_languages()
{
    global $bingClientID;
    global $bingClientSecret;
    try {
        $token = translate_gettoken();
        if (empty($token)) {
            return false;
        }
        $url = 'http://api.microsofttranslator.com/v2/Ajax.svc/GetLanguagesForTranslate?appId=';
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($ch, CURLOPT_HTTPHEADER, array('Authorization: Bearer ' . $token));
        $result = curl_exec($ch);
        curl_close($ch);
        $languages = json_decode(removeBOM($result));
        $languagestring = '["' . implode('","', $languages) . '"]';
        $url = 'http://api.microsofttranslator.com/v2/Ajax.svc/GetLanguageNames?locale=en&appId=&languageCodes=' . urlencode($languagestring);
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($ch, CURLOPT_HTTPHEADER, array('Authorization: Bearer ' . $token));
        $result = curl_exec($ch);
        curl_close($ch);
        $result = json_decode(removeBOM($result));
        $return = array();
        foreach ($result as $id => $value) {
            $return[$languages[$id]] = $value;
        }
        return $return;
    } catch (Exception $e) {
        return false;
    }
}

/**
* This function imports the old CSV data from 1.50 to 1.87 or older. Starting with 1.90 (DBVersion 143) there is an XML format instead
*
* @param array $sFullFilepath
* @returns array Information of imported questions/answers/etc.
*/
function CSVImportSurvey($sFullFilepath, $iDesiredSurveyId = NULL, $bTranslateLinks = true)
{
    Yii::app()->loadHelper('database');
    $clang = Yii::app()->lang;
    $handle = fopen($sFullFilepath, "r");
    while (!feof($handle)) {
        $buffer = fgets($handle);
        $bigarray[] = $buffer;
    }
    fclose($handle);
    $aIgnoredAnswers = array();
    $aSQIDReplacements = array();
    $aLIDReplacements = array();
    $aGIDReplacements = array();
    $substitutions = array();
    $aQuotaReplacements = array();
    $importresults['error'] = false;
    $importresults['importwarnings'] = array();
    $importresults['question_attributes'] = 0;
    if (isset($bigarray[0])) {
        $bigarray[0] = removeBOM($bigarray[0]);
    }
    // Now we try to determine the dataformat of the survey file.
    $importversion = 0;
    if (isset($bigarray[1]) && isset($bigarray[4]) && substr($bigarray[1], 0, 22) == "# SURVEYOR SURVEY DUMP") {
        $importversion = 100;
        // Version 0.99 or  1.0 file
    } elseif (substr($bigarray[0], 0, 24) == "# LimeSurvey Survey Dump" || substr($bigarray[0], 0, 25) == "# PHPSurveyor Survey Dump") {
        // Seems to be a >1.0 version file - these files carry the version information to read in line two
        $importversion = substr($bigarray[1], 12, 3);
    } else {
        $importresults['error'] = $clang->gT("This file is not a LimeSurvey survey file. Import failed.") . "\n";
        return $importresults;
    }
    if ((int) $importversion < 112) {
        $importresults['error'] = $clang->gT("This file is too old. Only files from LimeSurvey version 1.50 (DBVersion 112) and newer are supported.");
        return $importresults;
    }
    // okay.. now lets drop the first 9 lines and get to the data
    // This works for all versions
    for ($i = 0; $i < 9; $i++) {
        unset($bigarray[$i]);
    }
    $bigarray = array_values($bigarray);
    //SURVEYS
    if (array_search("# GROUPS TABLE\n", $bigarray)) {
        $stoppoint = array_search("# GROUPS TABLE\n", $bigarray);
    } elseif (array_search("# GROUPS TABLE\r\n", $bigarray)) {
        $stoppoint = array_search("# GROUPS TABLE\r\n", $bigarray);
    }
    for ($i = 0; $i <= $stoppoint + 1; $i++) {
        if ($i < $stoppoint - 2) {
            $surveyarray[] = $bigarray[$i];
        }
        unset($bigarray[$i]);
    }
    $bigarray = array_values($bigarray);
    //GROUPS
    if (array_search("# QUESTIONS TABLE\n", $bigarray)) {
        $stoppoint = array_search("# QUESTIONS TABLE\n", $bigarray);
    } elseif (array_search("# QUESTIONS TABLE\r\n", $bigarray)) {
        $stoppoint = array_search("# QUESTIONS TABLE\r\n", $bigarray);
    } else {
        $stoppoint = count($bigarray) - 1;
    }
    for ($i = 0; $i <= $stoppoint + 1; $i++) {
        if ($i < $stoppoint - 2) {
            $grouparray[] = $bigarray[$i];
        }
        unset($bigarray[$i]);
    }
    $bigarray = array_values($bigarray);
    //QUESTIONS
    if (array_search("# ANSWERS TABLE\n", $bigarray)) {
        $stoppoint = array_search("# ANSWERS TABLE\n", $bigarray);
    } elseif (array_search("# ANSWERS TABLE\r\n", $bigarray)) {
        $stoppoint = array_search("# ANSWERS TABLE\r\n", $bigarray);
    } else {
        $stoppoint = count($bigarray) - 1;
    }
    for ($i = 0; $i <= $stoppoint + 1; $i++) {
        if ($i < $stoppoint - 2) {
            $questionarray[] = $bigarray[$i];
        }
        unset($bigarray[$i]);
    }
    $bigarray = array_values($bigarray);
    //ANSWERS
    if (array_search("# CONDITIONS TABLE\n", $bigarray)) {
        $stoppoint = array_search("# CONDITIONS TABLE\n", $bigarray);
    } elseif (array_search("# CONDITIONS TABLE\r\n", $bigarray)) {
        $stoppoint = array_search("# CONDITIONS TABLE\r\n", $bigarray);
    } else {
        $stoppoint = count($bigarray) - 1;
//.........这里部分代码省略.........

 /**
  * import from csv
  */
 function import($iSurveyId)
 {
     $clang = $this->getController()->lang;
     $iSurveyId = (int) $iSurveyId;
     if (!Permission::model()->hasSurveyPermission($iSurveyId, 'tokens', 'import')) {
         Yii::app()->session['flashmessage'] = $clang->gT("You do not have sufficient rights to access this page.");
         $this->getController()->redirect(array("/admin/survey/sa/view/surveyid/{$iSurveyId}"));
     }
     // CHECK TO SEE IF A TOKEN TABLE EXISTS FOR THIS SURVEY
     $bTokenExists = tableExists('{{tokens_' . $iSurveyId . '}}');
     if (!$bTokenExists) {
         self::_newtokentable($iSurveyId);
     }
     App()->getClientScript()->registerScriptFile(Yii::app()->getConfig('adminscripts') . 'tokensimport.js');
     $aEncodings = aEncodingsArray();
     if (Yii::app()->request->getPost('submit')) {
         if (Yii::app()->request->getPost('csvcharset') && Yii::app()->request->getPost('csvcharset')) {
             $uploadcharset = Yii::app()->request->getPost('csvcharset');
             if (!array_key_exists($uploadcharset, $aEncodings)) {
                 $uploadcharset = 'auto';
             }
             $filterduplicatetoken = Yii::app()->request->getPost('filterduplicatetoken') && Yii::app()->request->getPost('filterduplicatetoken') == 'on';
             $filterblankemail = Yii::app()->request->getPost('filterblankemail') && Yii::app()->request->getPost('filterblankemail') == 'on';
         }
         $attrfieldnames = getAttributeFieldNames($iSurveyId);
         $duplicatelist = array();
         $invalidemaillist = array();
         $invalidformatlist = array();
         $firstline = array();
         $sPath = Yii::app()->getConfig('tempdir');
         $sFileTmpName = $_FILES['the_file']['tmp_name'];
         $sFilePath = $sPath . '/' . randomChars(20);
         if (!@move_uploaded_file($sFileTmpName, $sFilePath)) {
             $aData['sError'] = $clang->gT("Upload file not found. Check your permissions and path ({$sFilePath}) for the upload directory");
             $aData['aEncodings'] = $aEncodings;
             $aData['iSurveyId'] = $aData['surveyid'] = $iSurveyId;
             $aData['thissurvey'] = getSurveyInfo($iSurveyId);
             $this->_renderWrappedTemplate('token', array('tokenbar', 'csvupload'), $aData);
         } else {
             $xz = 0;
             $recordcount = 0;
             $xv = 0;
             // This allows to read file with MAC line endings too
             @ini_set('auto_detect_line_endings', true);
             // open it and trim the ednings
             $tokenlistarray = file($sFilePath);
             $sBaseLanguage = Survey::model()->findByPk($iSurveyId)->language;
             if (!Yii::app()->request->getPost('filterduplicatefields') || Yii::app()->request->getPost('filterduplicatefields') && count(Yii::app()->request->getPost('filterduplicatefields')) == 0) {
                 $filterduplicatefields = array('firstname', 'lastname', 'email');
             } else {
                 $filterduplicatefields = Yii::app()->request->getPost('filterduplicatefields');
             }
             $separator = returnGlobal('separator');
             foreach ($tokenlistarray as $buffer) {
                 $buffer = @mb_convert_encoding($buffer, "UTF-8", $uploadcharset);
                 if ($recordcount == 0) {
                     // Parse first line (header) from CSV
                     $buffer = removeBOM($buffer);
                     // We alow all field except tid because this one is really not needed.
                     $allowedfieldnames = array('participant_id', 'firstname', 'lastname', 'email', 'emailstatus', 'token', 'language', 'blacklisted', 'sent', 'remindersent', 'remindercount', 'validfrom', 'validuntil', 'completed', 'usesleft');
                     $allowedfieldnames = array_merge($attrfieldnames, $allowedfieldnames);
                     // Some header don't have same column name
                     $aReplacedFields = array('invited' => 'sent');
                     switch ($separator) {
                         case 'comma':
                             $separator = ',';
                             break;
                         case 'semicolon':
                             $separator = ';';
                             break;
                         default:
                             $comma = substr_count($buffer, ',');
                             $semicolon = substr_count($buffer, ';');
                             if ($semicolon > $comma) {
                                 $separator = ';';
                             } else {
                                 $separator = ',';
                             }
                     }
                     $firstline = str_getcsv($buffer, $separator, '"');
                     $firstline = array_map('trim', $firstline);
                     $ignoredcolumns = array();
                     // Now check the first line for invalid fields
                     foreach ($firstline as $index => $fieldname) {
                         $firstline[$index] = preg_replace("/(.*) <[^,]*>\$/", "\$1", $fieldname);
                         $fieldname = $firstline[$index];
                         if (!in_array($fieldname, $allowedfieldnames)) {
                             $ignoredcolumns[] = $fieldname;
                         }
                         if (array_key_exists($fieldname, $aReplacedFields)) {
                             $firstline[$index] = $aReplacedFields[$fieldname];
                         }
                     }
                     if (!in_array('firstname', $firstline) || !in_array('lastname', $firstline) || !in_array('email', $firstline)) {
                         $recordcount = count($tokenlistarray);
                         break;
                     }
//.........这里部分代码省略.........

        }
        if (!isset($_POST['filterduplicatefields']) || (isset($_POST['filterduplicatefields']) && count($_POST['filterduplicatefields'])==0))
        {
            $filterduplicatefields=array('firstname','lastname','email');
        } else {
            $filterduplicatefields=$_POST['filterduplicatefields'];
        }
        $separator = returnglobal('separator');
        foreach ($tokenlistarray as $buffer)
        {
            $buffer=@mb_convert_encoding($buffer,"UTF-8",$uploadcharset);
            $firstname = ""; $lastname = ""; $email = ""; $emailstatus="OK"; $token = ""; $language=""; $attribute1=""; $attribute2=""; //Clear out values from the last path, in case the next line is missing a value
            if ($recordcount==0)
            {
                // Pick apart the first line
                $buffer=removeBOM($buffer);
                $allowedfieldnames=array('firstname','lastname','email','emailstatus','token','language', 'validfrom', 'validuntil', 'usesleft');
                $allowedfieldnames=array_merge($attrfieldnames,$allowedfieldnames);

                switch ($separator) {
                    case 'comma':
                        $separator = ',';
                        break;
                    case 'semicolon':
                        $separator = ';';
                        break;
                    default:
                        $comma = substr_count($buffer,',');
                        $semicolon = substr_count($buffer,';');
                        if ($semicolon>$comma) $separator = ';'; else $separator = ',';
                }

 function uploadCSV()
 {
     $clang = $this->getController()->lang;
     unset(Yii::app()->session['summary']);
     $characterset = Yii::app()->request->getPost('characterset');
     $separator = Yii::app()->request->getPost('separatorused');
     $newarray = Yii::app()->request->getPost('newarray');
     $mappedarray = Yii::app()->request->getPost('mappedarray', false);
     $filterblankemails = Yii::app()->request->getPost('filterbea');
     $overwrite = Yii::app()->request->getPost('overwrite');
     $sFilePath = Yii::app()->getConfig('tempdir') . '/' . basename(Yii::app()->request->getPost('fullfilepath'));
     $errorinupload = "";
     $recordcount = 0;
     $mandatory = 0;
     $mincriteria = 0;
     $imported = 0;
     $dupcount = 0;
     $overwritten = 0;
     $dupreason = "nameemail";
     //Default duplicate comparison method
     $duplicatelist = array();
     $invalidemaillist = array();
     $invalidformatlist = array();
     $invalidattribute = array();
     $invalidparticipantid = array();
     $aGlobalErrors = array();
     /* If no mapped array */
     if (!$mappedarray) {
         $mappedarray = array();
     }
     /* Adjust system settings to read file with MAC line endings */
     @ini_set('auto_detect_line_endings', true);
     /* Open the uploaded file into an array */
     $tokenlistarray = file($sFilePath);
     // open it and trim the endings
     $separator = Yii::app()->request->getPost('separatorused');
     $uploadcharset = Yii::app()->request->getPost('characterset');
     /* The $newarray contains a list of fields that will be used
        to create new attributes */
     if (!empty($newarray)) {
         /* Create a new entry in the lime_participant_attribute_names table,
            and it's associated lime_participant_attribute_names_lang table
            for each NEW attribute being created in this import process */
         foreach ($newarray as $key => $value) {
             $aData = array('attribute_type' => 'TB', 'attribute_name' => $value, 'visible' => 'FALSE');
             $insertid = ParticipantAttributeName::model()->storeAttributeCSV($aData);
             /* Keep a record of the attribute_id for this new attribute
                in the $mappedarray string. For example, if the new attribute
                has attribute_id of 35 and is called "gender",
                $mappedarray['35']='gender' */
             $mappedarray[$insertid] = $value;
         }
     }
     if (!isset($uploadcharset)) {
         $uploadcharset = 'auto';
     }
     foreach ($tokenlistarray as $buffer) {
         //Iterate through the CSV file line by line
         $buffer = @mb_convert_encoding($buffer, "UTF-8", $uploadcharset);
         $firstname = "";
         $lastname = "";
         $email = "";
         $language = "";
         if ($recordcount == 0) {
             //The first time we iterate through the file we look at the very
             //first line, which contains field names, not values to import
             // Pick apart the first line
             $buffer = removeBOM($buffer);
             $attrid = ParticipantAttributeName::model()->getAttributeID();
             $allowedfieldnames = array('participant_id', 'firstname', 'lastname', 'email', 'language', 'blacklisted');
             $aFilterDuplicateFields = array('firstname', 'lastname', 'email');
             if (!empty($mappedarray)) {
                 foreach ($mappedarray as $key => $value) {
                     array_push($allowedfieldnames, strtolower($value));
                 }
             }
             //For Attributes
             switch ($separator) {
                 case 'comma':
                     $separator = ',';
                     break;
                 case 'semicolon':
                     $separator = ';';
                     break;
                 default:
                     $comma = substr_count($buffer, ',');
                     $semicolon = substr_count($buffer, ';');
                     if ($semicolon > $comma) {
                         $separator = ';';
                     } else {
                         $separator = ',';
                     }
             }
             $firstline = convertCSVRowToArray($buffer, $separator, '"');
             $firstline = array_map('trim', $firstline);
             $ignoredcolumns = array();
             //now check the first line for invalid fields
             foreach ($firstline as $index => $fieldname) {
                 $firstline[$index] = preg_replace("/(.*) <[^,]*>\$/", "\$1", $fieldname);
                 $fieldname = $firstline[$index];
//.........这里部分代码省略.........

 c("tl{$x}")->caption = c("tn{$x}")->caption = basename($gfn);
 $name = basename($gfn);
 $fn = $gfn;
 $name = basename($fn2);
 $data = file_get_contents($gfn);
 $data1 = $data;
 c("tmemo{$x}")->text = removeBOM($data);
 if ($data == c("tmemo{$x}")->text and $data[0] . $data[1] == "яю") {
     $coders = "ucs-2 Little Endian";
     $data = iconv('ucs-2', 'windows-1251', $data);
     c("tmemo{$x}")->text = $data;
     $text1 = c("tmemo{$x}")->text;
 }
 if ($data != c("tmemo{$x}")->text and $data[0] . $data[1] != "юя" and $data[0] . $data[1] != "яю") {
     $coders = "utf-8";
     c("tmemo{$x}")->text = iconv('utf-8', 'windows-1251', removeBOM($data));
     if (filesize($gfn) > 0 and $data[0] . $data[1] . $data[2] != "п»ї") {
         c("tmemo{$x}")->text = $data;
         $coders = "windows-1251";
         myFunc11();
     }
     $text1 = c("tmemo{$x}")->text;
 }
 if ($data == c("tmemo{$x}")->text and $data[0] . $data[1] == "юя") {
     $coders = "ucs-2 Big Endian";
     $data = iconv('ucs-2', 'windows-1251', $data);
     c("tmemo{$x}")->text = $data;
     $text1 = c("memo{$x}")->text;
 }
 if ($data == c("tmemo{$x}")->text and $data1[1] . $data1[0] != "яю" and $data1[1] . $data1[0] != "юя") {
     $text1 = c("tmemo{$x}")->text;

 /**
  * import from csv
  */
 public function import($iSurveyId)
 {
     $aData = array();
     $iSurveyId = (int) $iSurveyId;
     if (!Permission::model()->hasSurveyPermission($iSurveyId, 'tokens', 'import')) {
         Yii::app()->session['flashmessage'] = gT("You do not have permission to access this page.");
         $this->getController()->redirect(array("/admin/survey/sa/view/surveyid/{$iSurveyId}"));
     }
     // CHECK TO SEE IF A TOKEN TABLE EXISTS FOR THIS SURVEY
     $bTokenExists = tableExists('{{tokens_' . $iSurveyId . '}}');
     if (!$bTokenExists) {
         self::_newtokentable($iSurveyId);
     }
     $surveyinfo = Survey::model()->findByPk($iSurveyId)->surveyinfo;
     $aData['sidemenu']['state'] = false;
     $aData["surveyinfo"] = $surveyinfo;
     $aData['title_bar']['title'] = $surveyinfo['surveyls_title'] . "(" . gT("ID") . ":" . $iSurveyId . ")";
     $aData['sidemenu']["token_menu"] = TRUE;
     $aData['token_bar']['closebutton']['url'] = 'admin/tokens/sa/index/surveyid/' . $iSurveyId;
     $this->registerScriptFile('ADMIN_SCRIPT_PATH', 'tokensimport.js');
     $aEncodings = aEncodingsArray();
     if (Yii::app()->request->isPostRequest) {
         $sUploadCharset = Yii::app()->request->getPost('csvcharset');
         if (!array_key_exists($sUploadCharset, $aEncodings)) {
             $sUploadCharset = 'auto';
         }
         $bFilterDuplicateToken = Yii::app()->request->getPost('filterduplicatetoken');
         $bFilterBlankEmail = Yii::app()->request->getPost('filterblankemail');
         $bAllowInvalidEmail = Yii::app()->request->getPost('allowinvalidemail');
         $aAttrFieldNames = getAttributeFieldNames($iSurveyId);
         $aDuplicateList = array();
         $aInvalidTokenList = array();
         $aInvalidEmailList = array();
         $aInvalidFormatList = array();
         $aModelErrorList = array();
         $aFirstLine = array();
         $oFile = CUploadedFile::getInstanceByName("the_file");
         $sPath = Yii::app()->getConfig('tempdir');
         $sFileName = $sPath . '/' . randomChars(20);
         if ($_FILES['the_file']['error'] == 1 || $_FILES['the_file']['error'] == 2) {
             Yii::app()->setFlashMessage(sprintf(gT("Sorry, this file is too large. Only files up to %01.2f MB are allowed."), getMaximumFileUploadSize() / 1024 / 1024), 'error');
         } elseif (strtolower($oFile->getExtensionName()) != 'csv') {
             Yii::app()->setFlashMessage(gT("Only CSV files are allowed."), 'error');
         } elseif (!@$oFile->saveAs($sFileName)) {
             Yii::app()->setFlashMessage(sprintf(gT("Upload file not found. Check your permissions and path (%s) for the upload directory"), $sPath), 'error');
         } else {
             $iRecordImported = 0;
             $iRecordCount = 0;
             $iRecordOk = 0;
             $iInvalidEmailCount = 0;
             // Count invalid email imported
             // This allows to read file with MAC line endings too
             @ini_set('auto_detect_line_endings', true);
             // open it and trim the ednings
             $aTokenListArray = file($sFileName);
             $sBaseLanguage = Survey::model()->findByPk($iSurveyId)->language;
             if (!Yii::app()->request->getPost('filterduplicatefields') || Yii::app()->request->getPost('filterduplicatefields') && count(Yii::app()->request->getPost('filterduplicatefields')) == 0) {
                 $aFilterDuplicateFields = array('firstname', 'lastname', 'email');
             } else {
                 $aFilterDuplicateFields = Yii::app()->request->getPost('filterduplicatefields');
             }
             $sSeparator = Yii::app()->request->getPost('separator');
             $aMissingAttrFieldName = $aInvalideAttrFieldName = array();
             foreach ($aTokenListArray as $buffer) {
                 $buffer = @mb_convert_encoding($buffer, "UTF-8", $sUploadCharset);
                 if ($iRecordCount == 0) {
                     // Parse first line (header) from CSV
                     $buffer = removeBOM($buffer);
                     // We alow all field except tid because this one is really not needed.
                     $aAllowedFieldNames = Token::model($iSurveyId)->tableSchema->getColumnNames();
                     if (($kTid = array_search('tid', $aAllowedFieldNames)) !== false) {
                         unset($aAllowedFieldNames[$kTid]);
                     }
                     // Some header don't have same column name
                     $aReplacedFields = array('invited' => 'sent', 'reminded' => 'remindersent');
                     switch ($sSeparator) {
                         case 'comma':
                             $sSeparator = ',';
                             break;
                         case 'semicolon':
                             $sSeparator = ';';
                             break;
                         default:
                             $comma = substr_count($buffer, ',');
                             $semicolon = substr_count($buffer, ';');
                             if ($semicolon > $comma) {
                                 $sSeparator = ';';
                             } else {
                                 $sSeparator = ',';
                             }
                     }
                     $aFirstLine = str_getcsv($buffer, $sSeparator, '"');
                     $aFirstLine = array_map('trim', $aFirstLine);
                     $aIgnoredColumns = array();
                     // Now check the first line for invalid fields
                     foreach ($aFirstLine as $index => $sFieldname) {
                         $aFirstLine[$index] = preg_replace("/(.*) <[^,]*>\$/", "\$1", $sFieldname);
//.........这里部分代码省略.........

/**
* This function imports the old CSV data from 1.50 to 1.87 or older. Starting with 1.90 (DBVersion 143) there is an XML format instead
*
* @param array $sFullFilepath
* @returns array Information of imported questions/answers/etc.
*/
function CSVImportSurvey($sFullFilepath,$iDesiredSurveyId=NULL)
{
    global $dbprefix, $connect, $timeadjust, $clang;

    $handle = fopen($sFullFilepath, "r");
    while (!feof($handle))
    {

        $buffer = fgets($handle);
        $bigarray[] = $buffer;
    }
    fclose($handle);

    $aIgnoredAnswers=array();
    $aSQIDReplacements=array();
    $aLIDReplacements=array();
    $aGIDReplacements=array();
    $substitutions=array();
    $aQuotaReplacements=array();
    $importresults['error']=false;
    $importresults['importwarnings']=array();
    $importresults['question_attributes']=0;

    if (isset($bigarray[0])) $bigarray[0]=removeBOM($bigarray[0]);

    // Now we try to determine the dataformat of the survey file.
    $importversion=0;
    if (isset($bigarray[1]) && isset($bigarray[4])&& (substr($bigarray[1], 0, 22) == "# SURVEYOR SURVEY DUMP"))
    {
        $importversion = 100;  // Version 0.99 or  1.0 file
    }
    elseif
    (substr($bigarray[0], 0, 24) == "# LimeSurvey Survey Dump" || substr($bigarray[0], 0, 25) == "# PHPSurveyor Survey Dump")
    {  // Seems to be a >1.0 version file - these files carry the version information to read in line two
        $importversion=substr($bigarray[1], 12, 3);
    }
    else    // unknown file - show error message
    {
        $importresults['error'] = $clang->gT("This file is not a LimeSurvey survey file. Import failed.")."\n";
        return $importresults;
    }

    if  ((int)$importversion<112)
    {
        $importresults['error'] = $clang->gT("This file is too old. Only files from LimeSurvey version 1.50 (DBVersion 112) and newer are supported.");
        return $importresults;
    }

    // okay.. now lets drop the first 9 lines and get to the data
    // This works for all versions
    for ($i=0; $i<9; $i++)
    {
        unset($bigarray[$i]);
    }
    $bigarray = array_values($bigarray);


    //SURVEYS
    if (array_search("# GROUPS TABLE\n", $bigarray))
    {
        $stoppoint = array_search("# GROUPS TABLE\n", $bigarray);
    }
    elseif (array_search("# GROUPS TABLE\r\n", $bigarray))
    {
        $stoppoint = array_search("# GROUPS TABLE\r\n", $bigarray);
    }
    for ($i=0; $i<=$stoppoint+1; $i++)
    {
        if ($i<$stoppoint-2) {$surveyarray[] = $bigarray[$i];}
        unset($bigarray[$i]);
    }
    $bigarray = array_values($bigarray);

    //GROUPS
    if (array_search("# QUESTIONS TABLE\n", $bigarray))
    {
        $stoppoint = array_search("# QUESTIONS TABLE\n", $bigarray);
    }
    elseif (array_search("# QUESTIONS TABLE\r\n", $bigarray))
    {
        $stoppoint = array_search("# QUESTIONS TABLE\r\n", $bigarray);
    }
    else
    {
        $stoppoint = count($bigarray)-1;
    }
    for ($i=0; $i<=$stoppoint+1; $i++)
    {
        if ($i<$stoppoint-2) {$grouparray[] = $bigarray[$i];}
        unset($bigarray[$i]);
    }
    $bigarray = array_values($bigarray);

    //QUESTIONS
//.........这里部分代码省略.........

                $packer = new JavaScriptPacker($script, $encoding, $fast_decode, $special_char);
                $strFiles[$key] = $packer->pack();
            }
        }
        break;
}
//compress CSS files...
require_once 'includes/minify_css.php';
$cssFiles = array();
foreach ($cssSrc as $key => $src) {
    $cssFiles[$key] = array();
    $cssFiles[$key]['uncompressed'] = $src;
    $cssFiles[$key]['compressed'] = Minify_CSS_Compressor::process($src);
}
$licenseFile['jxlib'] = removeBOM(file_get_contents('src' . DS . 'jxlib' . DS . 'Source' . DS . 'license.js'));
$licenseFile['core'] = '/*' . removeBOM(file_get_contents('src' . DS . 'core' . DS . 'Source' . DS . 'license.txt')) . '*/';
//add license file(s)
foreach ($strFiles as $key => $value) {
    if (strpos($key, 'jxlib')) {
        $strFiles[$key] = $licenseFile['jxlib'] . "\n" . $value;
    } else {
        $strFiles[$key] = $licenseFile['core'] . "\n" . $value;
    }
}
function guid()
{
    $g = '';
    if (function_exists('com_create_guid')) {
        $g = com_create_guid();
    } else {
        mt_srand((double) microtime() * 10000);

 /**
  * import from csv
  */
 function import($iSurveyId)
 {
     $iSurveyId = (int) $iSurveyId;
     if (!Permission::model()->hasSurveyPermission($iSurveyId, 'tokens', 'import')) {
         Yii::app()->session['flashmessage'] = gT("You do not have sufficient rights to access this page.");
         $this->getController()->redirect(array("/admin/survey/sa/view/surveyid/{$iSurveyId}"));
     }
     // CHECK TO SEE IF A TOKEN TABLE EXISTS FOR THIS SURVEY
     $bTokenExists = tableExists('{{tokens_' . $iSurveyId . '}}');
     if (!$bTokenExists) {
         self::_newtokentable($iSurveyId);
     }
     App()->getClientScript()->registerScriptFile(Yii::app()->getConfig('adminscripts') . 'tokensimport.js');
     $aEncodings = aEncodingsArray();
     if (Yii::app()->request->isPostRequest) {
         $sUploadCharset = Yii::app()->request->getPost('csvcharset');
         if (!array_key_exists($sUploadCharset, $aEncodings)) {
             $sUploadCharset = 'auto';
         }
         $bFilterDuplicateToken = Yii::app()->request->getPost('filterduplicatetoken');
         $bFilterBlankEmail = Yii::app()->request->getPost('filterblankemail');
         $bAllowInvalidEmail = Yii::app()->request->getPost('allowinvalidemail');
         $aAttrFieldNames = getAttributeFieldNames($iSurveyId);
         $aDuplicateList = array();
         $aInvalidEmailList = array();
         $aInvalidFormatList = array();
         $aModelErrorList = array();
         $aFirstLine = array();
         $oFile = CUploadedFile::getInstanceByName("the_file");
         $sPath = Yii::app()->getConfig('tempdir');
         $sFileName = $sPath . '/' . randomChars(20);
         //$sFileTmpName=$oFile->getTempName();
         /* More way to validate CSV ?
            $aCsvMimetypes = array(
                'text/csv',
                'text/plain',
                'application/csv',
                'text/comma-separated-values',
                'application/excel',
                'application/vnd.ms-excel',
                'application/vnd.msexcel',
                'text/anytext',
                'application/octet-stream',
                'application/txt',
            );
            */
         if (strtolower($oFile->getExtensionName()) != 'csv') {
             Yii::app()->setFlashMessage(gT("Only CSV files are allowed."), 'error');
         } elseif (!@$oFile->saveAs($sFileName)) {
             Yii::app()->setFlashMessage(sprintf(gT("Upload file not found. Check your permissions and path (%s) for the upload directory"), $sPath), 'error');
         } else {
             $iRecordImported = 0;
             $iRecordCount = 0;
             $iRecordOk = 0;
             $iInvalidEmailCount = 0;
             // Count invalid email imported
             // This allows to read file with MAC line endings too
             @ini_set('auto_detect_line_endings', true);
             // open it and trim the ednings
             $aTokenListArray = file($sFileName);
             $sBaseLanguage = Survey::model()->findByPk($iSurveyId)->language;
             if (!Yii::app()->request->getPost('filterduplicatefields') || Yii::app()->request->getPost('filterduplicatefields') && count(Yii::app()->request->getPost('filterduplicatefields')) == 0) {
                 $aFilterDuplicateFields = array('firstname', 'lastname', 'email');
             } else {
                 $aFilterDuplicateFields = Yii::app()->request->getPost('filterduplicatefields');
             }
             $sSeparator = Yii::app()->request->getPost('separator');
             foreach ($aTokenListArray as $buffer) {
                 $buffer = @mb_convert_encoding($buffer, "UTF-8", $sUploadCharset);
                 if ($iRecordCount == 0) {
                     // Parse first line (header) from CSV
                     $buffer = removeBOM($buffer);
                     // We alow all field except tid because this one is really not needed.
                     $aAllowedFieldNames = Token::model($iSurveyId)->tableSchema->getColumnNames();
                     if (($kTid = array_search('tid', $aAllowedFieldNames)) !== false) {
                         unset($aAllowedFieldNames[$kTid]);
                     }
                     // Some header don't have same column name
                     $aReplacedFields = array('invited' => 'sent', 'reminded' => 'remindersent');
                     switch ($sSeparator) {
                         case 'comma':
                             $sSeparator = ',';
                             break;
                         case 'semicolon':
                             $sSeparator = ';';
                             break;
                         default:
                             $comma = substr_count($buffer, ',');
                             $semicolon = substr_count($buffer, ';');
                             if ($semicolon > $comma) {
                                 $sSeparator = ';';
                             } else {
                                 $sSeparator = ',';
                             }
                     }
                     $aFirstLine = str_getcsv($buffer, $sSeparator, '"');
                     $aFirstLine = array_map('trim', $aFirstLine);
//.........这里部分代码省略.........

 /**
  * import from csv
  */
 function import($iSurveyId)
 {
     // CHECK TO SEE IF A TOKEN TABLE EXISTS FOR THIS SURVEY
     $bTokenExists = tableExists('{{tokens_' . $iSurveyId . '}}');
     if (!$bTokenExists) {
         self::_newtokentable($iSurveyId);
     }
     $clang = $this->getController()->lang;
     $iSurveyId = (int) $iSurveyId;
     if (!hasSurveyPermission($iSurveyId, 'tokens', 'create')) {
         die('access denied');
     }
     $this->getController()->_js_admin_includes(Yii::app()->getConfig('adminscripts') . 'tokens.js');
     $aEncodings = array("armscii8" => $clang->gT("ARMSCII-8 Armenian"), "ascii" => $clang->gT("US ASCII"), "auto" => $clang->gT("Automatic"), "big5" => $clang->gT("Big5 Traditional Chinese"), "binary" => $clang->gT("Binary pseudo charset"), "cp1250" => $clang->gT("Windows Central European"), "cp1251" => $clang->gT("Windows Cyrillic"), "cp1256" => $clang->gT("Windows Arabic"), "cp1257" => $clang->gT("Windows Baltic"), "cp850" => $clang->gT("DOS West European"), "cp852" => $clang->gT("DOS Central European"), "cp866" => $clang->gT("DOS Russian"), "cp932" => $clang->gT("SJIS for Windows Japanese"), "dec8" => $clang->gT("DEC West European"), "eucjpms" => $clang->gT("UJIS for Windows Japanese"), "euckr" => $clang->gT("EUC-KR Korean"), "gb2312" => $clang->gT("GB2312 Simplified Chinese"), "gbk" => $clang->gT("GBK Simplified Chinese"), "geostd8" => $clang->gT("GEOSTD8 Georgian"), "greek" => $clang->gT("ISO 8859-7 Greek"), "hebrew" => $clang->gT("ISO 8859-8 Hebrew"), "hp8" => $clang->gT("HP West European"), "keybcs2" => $clang->gT("DOS Kamenicky Czech-Slovak"), "koi8r" => $clang->gT("KOI8-R Relcom Russian"), "koi8u" => $clang->gT("KOI8-U Ukrainian"), "latin1" => $clang->gT("cp1252 West European"), "latin2" => $clang->gT("ISO 8859-2 Central European"), "latin5" => $clang->gT("ISO 8859-9 Turkish"), "latin7" => $clang->gT("ISO 8859-13 Baltic"), "macce" => $clang->gT("Mac Central European"), "macroman" => $clang->gT("Mac West European"), "sjis" => $clang->gT("Shift-JIS Japanese"), "swe7" => $clang->gT("7bit Swedish"), "tis620" => $clang->gT("TIS620 Thai"), "ucs2" => $clang->gT("UCS-2 Unicode"), "ujis" => $clang->gT("EUC-JP Japanese"), "utf8" => $clang->gT("UTF-8 Unicode"));
     if (Yii::app()->request->getPost('submit')) {
         if (Yii::app()->request->getPost('csvcharset') && Yii::app()->request->getPost('csvcharset')) {
             $uploadcharset = Yii::app()->request->getPost('csvcharset');
             if (!array_key_exists($uploadcharset, $aEncodings)) {
                 $uploadcharset = 'auto';
             }
             $filterduplicatetoken = Yii::app()->request->getPost('filterduplicatetoken') && Yii::app()->request->getPost('filterduplicatetoken') == 'on';
             $filterblankemail = Yii::app()->request->getPost('filterblankemail') && Yii::app()->request->getPost('filterblankemail') == 'on';
         }
         $attrfieldnames = getAttributeFieldNames($iSurveyId);
         $duplicatelist = array();
         $invalidemaillist = array();
         $invalidformatlist = array();
         $firstline = array();
         $sPath = Yii::app()->getConfig('tempdir');
         $sFileName = $_FILES['the_file']['name'];
         $sFileTmpName = $_FILES['the_file']['tmp_name'];
         $sFilePath = $sPath . '/' . $sFileName;
         if (!@move_uploaded_file($sFileTmpName, $sFilePath)) {
             $aData['sError'] = $clang->gT("Upload file not found. Check your permissions and path ({$sFilePath}) for the upload directory");
             $aData['aEncodings'] = $aEncodings;
             $aData['iSurveyId'] = $aData['surveyid'] = $iSurveyId;
             $aData['thissurvey'] = getSurveyInfo($iSurveyId);
             $this->_renderWrappedTemplate('token', array('tokenbar', 'csvupload'), $aData);
         } else {
             $xz = 0;
             $recordcount = 0;
             $xv = 0;
             // This allows to read file with MAC line endings too
             @ini_set('auto_detect_line_endings', true);
             // open it and trim the ednings
             $tokenlistarray = file($sFilePath);
             $sBaseLanguage = Survey::model()->findByPk($iSurveyId)->language;
             if (!Yii::app()->request->getPost('filterduplicatefields') || Yii::app()->request->getPost('filterduplicatefields') && count(Yii::app()->request->getPost('filterduplicatefields')) == 0) {
                 $filterduplicatefields = array('firstname', 'lastname', 'email');
             } else {
                 $filterduplicatefields = Yii::app()->request->getPost('filterduplicatefields');
             }
             $separator = returnGlobal('separator');
             foreach ($tokenlistarray as $buffer) {
                 $buffer = @mb_convert_encoding($buffer, "UTF-8", $uploadcharset);
                 $firstname = "";
                 $lastname = "";
                 $email = "";
                 $emailstatus = "OK";
                 $token = "";
                 $language = "";
                 $attribute1 = "";
                 $attribute2 = "";
                 //Clear out values from the last path, in case the next line is missing a value
                 if ($recordcount == 0) {
                     // Pick apart the first line
                     $buffer = removeBOM($buffer);
                     $allowedfieldnames = array('firstname', 'lastname', 'email', 'emailstatus', 'token', 'language', 'validfrom', 'validuntil', 'usesleft');
                     $allowedfieldnames = array_merge($attrfieldnames, $allowedfieldnames);
                     switch ($separator) {
                         case 'comma':
                             $separator = ',';
                             break;
                         case 'semicolon':
                             $separator = ';';
                             break;
                         default:
                             $comma = substr_count($buffer, ',');
                             $semicolon = substr_count($buffer, ';');
                             if ($semicolon > $comma) {
                                 $separator = ';';
                             } else {
                                 $separator = ',';
                             }
                     }
                     $firstline = convertCSVRowToArray($buffer, $separator, '"');
                     $firstline = array_map('trim', $firstline);
                     $ignoredcolumns = array();
                     //now check the first line for invalid fields
                     foreach ($firstline as $index => $fieldname) {
                         $firstline[$index] = preg_replace("/(.*) <[^,]*>\$/", "\$1", $fieldname);
                         $fieldname = $firstline[$index];
                         if (!in_array($fieldname, $allowedfieldnames)) {
                             $ignoredcolumns[] = $fieldname;
                         }
                     }
                     if (!in_array('firstname', $firstline) || !in_array('lastname', $firstline) || !in_array('email', $firstline)) {
//.........这里部分代码省略.........

/**
 * 任意编码的字符串转为UTF-8(non BOM)，支持一些常见东方语言
 *
 * @param string $str 输入字符串
 * @return string 输出字符串
 */
function anystring2utf8($str)
{
    $encode = mb_detect_encoding($str, 'ASCII,UNICODE,UTF-8,CP936,BIG-5,EUC-TW');
    return removeBOM(!in_array($encode, array('UTF-8', 'ASCII')) ? iconv($encode, 'UTF-8//IGNORE', $str) : $str);
    //移除BOM的UTF-8
}

/**
 * Loads the specified file, optionally searching the include path, and stripping the Unicode Byte Order Mark (BOM), if
 * one is present.
 *
 * @param string    $filename
 * @param bool|true $useIncludePath
 * @return false|string `false` if the file is not found.
 */
function loadFile($filename, $useIncludePath = true)
{
    if ($useIncludePath) {
        if (!($filename = stream_resolve_include_path($filename))) {
            return false;
        }
    } else {
        if (!file_exists($filename)) {
            return false;
        }
    }
    return removeBOM(file_get_contents($filename));
}