Python util.find_duplicates函数代码示例

    def test_empty_input(self):
        def empty_gen():
            return
            yield

        for empty in [], (), '', set(), {}, empty_gen():
            self.assertEqual(find_duplicates(empty), set())

    def _build_mapping_file(self, samples):
        """Builds the combined mapping file for all samples
           Code modified slightly from qiime.util.MetadataMap.__add__"""
        conn_handler = SQLConnectionHandler()
        all_sample_ids = set()
        sql = """SELECT filepath_id, filepath
                 FROM qiita.filepath
                    JOIN qiita.prep_template_filepath USING (filepath_id)
                    JOIN qiita.prep_template_preprocessed_data
                        USING (prep_template_id)
                    JOIN qiita.preprocessed_processed_data
                        USING (preprocessed_data_id)
                    JOIN qiita.filepath_type USING (filepath_type_id)
                 WHERE processed_data_id = %s
                    AND filepath_type = 'qiime_map'
                 ORDER BY filepath_id DESC"""
        _id, fp = get_mountpoint('templates')[0]
        to_concat = []

        for pid, samples in viewitems(samples):
            if len(samples) != len(set(samples)):
                duplicates = find_duplicates(samples)
                raise QiitaDBError("Duplicate sample ids found: %s"
                                   % ', '.join(duplicates))
            # Get the QIIME mapping file
            qiime_map_fp = conn_handler.execute_fetchall(sql, (pid,))[0][1]
            # Parse the mapping file
            qiime_map = pd.read_csv(
                join(fp, qiime_map_fp), sep='\t', keep_default_na=False,
                na_values=['unknown'], index_col=False,
                converters=defaultdict(lambda: str))
            qiime_map.set_index('#SampleID', inplace=True, drop=True)
            qiime_map = qiime_map.loc[samples]

            duplicates = all_sample_ids.intersection(qiime_map.index)
            if duplicates or len(samples) != len(set(samples)):
                # Duplicate samples so raise error
                raise QiitaDBError("Duplicate sample ids found: %s"
                                   % ', '.join(duplicates))
            all_sample_ids.update(qiime_map.index)
            to_concat.append(qiime_map)

        merged_map = pd.concat(to_concat)

        cols = merged_map.columns.values.tolist()
        cols.remove('BarcodeSequence')
        cols.remove('LinkerPrimerSequence')
        cols.remove('Description')
        new_cols = ['BarcodeSequence', 'LinkerPrimerSequence']
        new_cols.extend(cols)
        new_cols.append('Description')
        merged_map = merged_map[new_cols]

        # Save the mapping file
        _, base_fp = get_mountpoint(self._table)[0]
        mapping_fp = join(base_fp, "%d_analysis_mapping.txt" % self._id)
        merged_map.to_csv(mapping_fp, index_label='#SampleID',
                          na_rep='unknown', sep='\t')

        self._add_file("%d_analysis_mapping.txt" % self._id, "plain_text")

    def test_empty_input(self):
        def empty_gen():
            raise StopIteration()
            yield

        for empty in [], (), '', set(), {}, empty_gen():
            self.assertEqual(find_duplicates(empty), set())

    def _validate(self, data, ids):
        """Validate the data array and IDs.

        Checks that the data is at least 1x1 in size, 2D, square, hollow, and
        contains only floats. Also checks that IDs are unique and that the
        number of IDs matches the number of rows/cols in the data array.

        Subclasses can override this method to perform different/more specific
        validation (e.g., see `DistanceMatrix`).

        Notes
        -----
        Accepts arguments instead of inspecting instance attributes to avoid
        creating an invalid dissimilarity matrix before raising an error.
        Otherwise, the invalid dissimilarity matrix could be used after the
        exception is caught and handled.

        """
        if 0 in data.shape:
            raise DissimilarityMatrixError("Data must be at least 1x1 in "
                                           "size.")
        if len(data.shape) != 2:
            raise DissimilarityMatrixError("Data must have exactly two "
                                           "dimensions.")
        if data.shape[0] != data.shape[1]:
            raise DissimilarityMatrixError("Data must be square (i.e., have "
                                           "the same number of rows and "
                                           "columns).")
        if data.dtype != np.double:
            raise DissimilarityMatrixError("Data must contain only floating "
                                           "point values.")
        if np.trace(data) != 0:
            raise DissimilarityMatrixError("Data must be hollow (i.e., the "
                                           "diagonal can only contain zeros).")
        duplicates = find_duplicates(ids)
        if duplicates:
            formatted_duplicates = ', '.join(repr(e) for e in duplicates)
            raise DissimilarityMatrixError("IDs must be unique. Found the "
                                           "following duplicate IDs: %s" %
                                           formatted_duplicates)
        if len(ids) != data.shape[0]:
            raise DissimilarityMatrixError("The number of IDs (%d) must match "
                                           "the number of rows/columns in the "
                                           "data (%d)." %
                                           (len(ids), data.shape[0]))

def _check_duplicated_columns(prep_cols, sample_cols):
    r"""Check for duplicated colums in the prep_cols and sample_cols

    Parameters
    ----------
    prep_cols : list of str
        Column names in the prep info file
    sample_cols : list of str
        Column names in the sample info file

    Raises
    ------
    QiitaDBColumnError
        If there are duplicated columns names in the sample and the prep
    """
    prep_cols.extend(sample_cols)
    dups = find_duplicates(prep_cols)
    if dups:
        raise qdb.exceptions.QiitaDBColumnError(
            'Duplicated column names in the sample and prep info '
            'files: %s. You need to delete that duplicated field' %
            ','.join(dups))

    def test_mixed_types(self):
        def gen():
            for e in 'a', 1, 'bc', 2, 'a', 2, 2, 3.0:
                yield e

        self.assertEqual(find_duplicates(gen()), set(['a', 2]))

 def test_all_duplicates(self):
     self.assertEqual(
         find_duplicates(('a', 'bc', 'bc', 'def', 'a', 'def', 'def')),
         set(['a', 'bc', 'def']))

 def test_many_duplicates(self):
     self.assertEqual(find_duplicates(['a', 'bc', 'bc', 'def', 'a']),
                      set(['a', 'bc']))

 def test_no_duplicates(self):
     self.assertEqual(find_duplicates(['a', 'bc', 'def', 'A']), set())

    def test_mixed_types(self):
        def gen():
            yield from ('a', 1, 'bc', 2, 'a', 2, 2, 3.0)

        self.assertEqual(find_duplicates(gen()), set(['a', 2]))

def load_template_to_dataframe(fn, index='sample_name'):
    """Load a sample/prep template or a QIIME mapping file into a data frame

    Parameters
    ----------
    fn : str or file-like object
        filename of the template to load, or an already open template file
    index : str, optional
        Defaults to 'sample_name'. The index to use in the loaded information

    Returns
    -------
    DataFrame
        Pandas dataframe with the loaded information

    Raises
    ------
    ValueError
        Empty file passed
    QiitaDBColumnError
        If the sample_name column is not present in the template.
    QiitaDBWarning
        When columns are dropped because they have no content for any sample.
    QiitaDBError
        When non UTF-8 characters are found in the file.
    QiitaDBDuplicateHeaderError
        If duplicate columns are present in the template

    Notes
    -----
    The index attribute of the DataFrame will be forced to be 'sample_name'
    and will be cast to a string. Additionally rows that start with a '\t'
    character will be ignored and columns that are empty will be removed. Empty
    sample names will be removed from the DataFrame.

    Column names are case-insensitive but will be lowercased on addition to
    the database

    Everything in the DataFrame will be read and managed as string
    """
    # Load in file lines
    holdfile = None
    with qdb.util.open_file(fn, newline=None,
                            encoding="utf8", errors='ignore') as f:
        holdfile = f.readlines()

    if not holdfile:
        raise ValueError('Empty file passed!')

    if index == "#SampleID":
        # We're going to parse a QIIME mapping file. We are going to first
        # parse it with the QIIME function so we can remove the comments
        # easily and make sure that QIIME will accept this as a mapping file
        data, headers, comments = _parse_mapping_file(holdfile)
        holdfile = ["%s\n" % '\t'.join(d) for d in data]
        holdfile.insert(0, "%s\n" % '\t'.join(headers))
        # The QIIME parser fixes the index and removes the #
        index = 'SampleID'

    # Strip all values in the cells in the input file
    for pos, line in enumerate(holdfile):
        cols = line.split('\t')
        if pos == 0 and index != 'SampleID':
            # get and clean the controlled columns
            ccols = {'sample_name'}
            ccols.update(qdb.metadata_template.constants.CONTROLLED_COLS)
            newcols = [
                c.lower().strip() if c.lower().strip() in ccols
                else c.strip()
                for c in cols]

            # while we are here, let's check for duplicate columns headers
            ncols = set(newcols)
            if len(ncols) != len(newcols):
                if '' in ncols:
                    raise ValueError(
                        'Your file has empty columns headers.')
                raise qdb.exceptions.QiitaDBDuplicateHeaderError(
                    find_duplicates(newcols))
        else:
            # .strip will remove odd chars, newlines, tabs and multiple
            # spaces but we need to read a new line at the end of the
            # line(+'\n')
            newcols = [d.strip(" \r\n") for d in cols]

        holdfile[pos] = '\t'.join(newcols) + '\n'

    # index_col:
    #   is set as False, otherwise it is cast as a float and we want a string
    # keep_default:
    #   is set as False, to avoid inferring empty/NA values with the defaults
    #   that Pandas has.
    # comment:
    #   using the tab character as "comment" we remove rows that are
    #   constituted only by delimiters i. e. empty rows.
    template = pd.read_csv(
        StringIO(''.join(holdfile)),
        sep='\t',
        dtype=str,
        encoding='utf-8',
#.........这里部分代码省略.........

    def _clean_validate_template(cls, md_template, study_id, restriction_dict,
                                 current_columns=None):
        """Takes care of all validation and cleaning of metadata templates

        Parameters
        ----------
        md_template : DataFrame
            The metadata template file contents indexed by sample ids
        study_id : int
            The study to which the metadata template belongs to.
        restriction_dict : dict of {str: Restriction}
            A dictionary with the restrictions that apply to the metadata
        current_columns : iterable of str, optional
            The current list of metadata columns

        Returns
        -------
        md_template : DataFrame
            Cleaned copy of the input md_template

        Raises
        ------
        QiitaDBColumnError
            If the sample names in md_template contains invalid names
        QiitaDBWarning
            If there are missing columns required for some functionality
        """
        cls._check_subclass()
        invalid_ids = qdb.metadata_template.util.get_invalid_sample_names(
            md_template.index)
        if invalid_ids:
            raise qdb.exceptions.QiitaDBColumnError(
                "The following sample names in the template contain invalid "
                "characters (only alphanumeric characters or periods are "
                "allowed): %s." % ", ".join(invalid_ids))

        if len(set(md_template.index)) != len(md_template.index):
            raise qdb.exceptions.QiitaDBDuplicateSamplesError(
                find_duplicates(md_template.index))

        # We are going to modify the md_template. We create a copy so
        # we don't modify the user one
        md_template = md_template.copy(deep=True)

        # Prefix the sample names with the study_id
        qdb.metadata_template.util.prefix_sample_names_with_id(md_template,
                                                               study_id)

        # In the database, all the column headers are lowercase
        md_template.columns = [c.lower() for c in md_template.columns]

        # Check that we don't have duplicate columns
        if len(set(md_template.columns)) != len(md_template.columns):
            raise qdb.exceptions.QiitaDBDuplicateHeaderError(
                find_duplicates(md_template.columns))

        # Check if we have the columns required for some functionality
        warning_msg = []
        columns = set(md_template.columns)
        if current_columns:
            columns.update(current_columns)
        for key, restriction in viewitems(restriction_dict):
            missing = set(restriction.columns).difference(columns)

            if missing:
                warning_msg.append(
                    "%s: %s" % (restriction.error_msg, ', '.join(missing)))

        if warning_msg:
            warnings.warn(
                "Some functionality will be disabled due to missing "
                "columns:\n\t%s.\nSee the Templates tutorial for a description"
                " of these fields." % ";\n\t".join(warning_msg),
                qdb.exceptions.QiitaDBWarning)

        return md_template

    def reindex(self, key=None, keys=None):
        """Reassign keys to sequences in the MSA.

        Parameters
        ----------
        key : callable or metadata key, optional
            If provided, defines a unique, hashable key for each sequence in
            the MSA. Can either be a callable accepting a single argument (each
            sequence) or a key into each sequence's ``metadata`` attribute.
        keys : iterable, optional
            An iterable of the same length as the number of sequences in the
            MSA. `keys` must contain unique, hashable elements. Each element
            will be used as the respective key for the sequences in the MSA.

        Raises
        ------
        ValueError
            If `key` and `keys` are both provided.
        ValueError
            If `keys` is not the same length as the number of sequences in the
            MSA.
        UniqueError
            If keys are not unique.

        See Also
        --------
        keys
        has_keys

        Notes
        -----
        If `key` or `keys` are not provided, keys will not be set and certain
        operations requiring keys will raise an ``OperationError``.

        Examples
        --------
        Create a ``TabularMSA`` object without keys:

        >>> from skbio import DNA, TabularMSA
        >>> seqs = [DNA('ACG', metadata={'id': 'a'}),
        ...         DNA('AC-', metadata={'id': 'b'})]
        >>> msa = TabularMSA(seqs)
        >>> msa.has_keys()
        False

        Set keys on the MSA, using each sequence's ID:

        >>> msa.reindex(key='id')
        >>> msa.has_keys()
        True
        >>> msa.keys
        array(['a', 'b'], dtype=object)

        Remove keys from the MSA:

        >>> msa.reindex()
        >>> msa.has_keys()
        False

        Alternatively, an iterable of keys may be passed via `keys`:

        >>> msa.reindex(keys=['a', 'b'])
        >>> msa.keys
        array(['a', 'b'], dtype=object)

        """
        if key is not None and keys is not None:
            raise ValueError(
                "Cannot use both `key` and `keys` at the same time.")

        keys_ = None
        if key is not None:
            keys_ = [resolve_key(seq, key) for seq in self._seqs]
        elif keys is not None:
            keys = list(keys)
            if len(keys) != len(self):
                raise ValueError(
                    "Number of elements in `keys` must match number of "
                    "sequences: %d != %d" % (len(keys), len(self)))
            keys_ = keys

        if keys_ is not None:
            # Hashability of keys is implicitly checked here.
            duplicates = find_duplicates(keys_)
            if duplicates:
                raise UniqueError(
                    "Keys must be unique. Duplicate keys: %r" % duplicates)

            # Create an immutable ndarray to ensure key invariants are
            # preserved. Use object dtype to preserve original key types. This
            # is important, for example, because np.array(['a', 42]) will
            # upcast to ['a', '42'].
            keys_ = np.array(keys_, dtype=object, copy=True)
            keys_.flags.writeable = False

        self._keys = keys_

    def _clean_validate_template(cls, md_template, study_id, restriction_dict):
        """Takes care of all validation and cleaning of metadata templates

        Parameters
        ----------
        md_template : DataFrame
            The metadata template file contents indexed by sample ids
        study_id : int
            The study to which the metadata template belongs to.
        restriction_dict : dict of {str: Restriction}
            A dictionary with the restrictions that apply to the metadata

        Returns
        -------
        md_template : DataFrame
            Cleaned copy of the input md_template

        Raises
        ------
        QiitaDBColumnError
            If the sample names in md_template contains invalid names
        QiitaDBWarning
            If there are missing columns required for some functionality
        """
        cls._check_subclass()
        invalid_ids = get_invalid_sample_names(md_template.index)
        if invalid_ids:
            raise QiitaDBColumnError("The following sample names in the "
                                     "template contain invalid characters "
                                     "(only alphanumeric characters or periods"
                                     " are allowed): %s." %
                                     ", ".join(invalid_ids))

        # We are going to modify the md_template. We create a copy so
        # we don't modify the user one
        md_template = deepcopy(md_template)

        # Prefix the sample names with the study_id
        prefix_sample_names_with_id(md_template, study_id)

        # In the database, all the column headers are lowercase
        md_template.columns = [c.lower() for c in md_template.columns]

        # Check that we don't have duplicate columns
        if len(set(md_template.columns)) != len(md_template.columns):
            raise QiitaDBDuplicateHeaderError(
                find_duplicates(md_template.columns))

        # Check if we have the columns required for some functionality
        warning_msg = []
        for key, restriction in viewitems(restriction_dict):
            missing = set(restriction.columns).difference(md_template)

            if missing:
                warning_msg.append(
                    "%s: %s" % (restriction.error_msg, ', '.join(missing)))

        if warning_msg:
            warnings.warn(
                "Some functionality will be disabled due to missing "
                "columns:\n\t%s.\nCheck https://github.com/biocore/qiita/wiki"
                "/Preparing-Qiita-template-files for a description of these "
                "fields." % ";\n\t".join(warning_msg),
                QiitaDBWarning)

        return md_template

#.........这里部分代码省略.........

    # index_col:
    #   is set as False, otherwise it is cast as a float and we want a string
    # keep_default:
    #   is set as False, to avoid inferring empty/NA values with the defaults
    #   that Pandas has.
    # na_values:
    #   the values that should be considered as empty
    # true_values:
    #   the values that should be considered "True" for boolean columns
    # false_values:
    #   the values that should be considered "False" for boolean columns
    # converters:
    #   ensure that sample names are not converted into any other types but
    #   strings and remove any trailing spaces. Don't let pandas try to guess
    #   the dtype of the other columns, force them to be a str.
    # comment:
    #   using the tab character as "comment" we remove rows that are
    #   constituted only by delimiters i. e. empty rows.
    try:
        template = pd.read_csv(StringIO(''.join(holdfile)), sep='\t',
                               encoding='utf-8', infer_datetime_format=True,
                               keep_default_na=False, na_values=NA_VALUES,
                               true_values=TRUE_VALUES,
                               false_values=FALSE_VALUES,
                               parse_dates=True, index_col=False, comment='\t',
                               mangle_dupe_cols=False, converters={
                                   index: lambda x: str(x).strip(),
                                   # required sample template information
                                   'physical_location': str,
                                   'sample_type': str,
                                   # collection_timestamp is not added here
                                   'host_subject_id': str,
                                   'description': str,
                                   # common prep template information
                                   'center_name': str,
                                   'center_projct_name': str})
    except UnicodeDecodeError:
        # Find row number and col number for utf-8 encoding errors
        headers = holdfile[0].strip().split('\t')
        errors = defaultdict(list)
        for row, line in enumerate(holdfile, 1):
            for col, cell in enumerate(line.split('\t')):
                try:
                    cell.encode('utf-8')
                except UnicodeError:
                    errors[headers[col]].append(row)
        lines = ['%s: row(s) %s' % (header, ', '.join(map(str, rows)))
                 for header, rows in viewitems(errors)]
        raise QiitaDBError('Non UTF-8 characters found in columns:\n' +
                           '\n'.join(lines))

    # Check that we don't have duplicate columns
    if len(set(template.columns)) != len(template.columns):
        raise QiitaDBDuplicateHeaderError(find_duplicates(template.columns))

    # let pandas infer the dtypes of these columns, if the inference is
    # not correct, then we have to raise an error
    columns_to_dtype = [(['latitude', 'longitude'], (np.int, np.float),
                         'integer or decimal'),
                        (['has_physical_specimen', 'has_extracted_data'],
                         np.bool_, 'boolean')]
    for columns, c_dtype, english_desc in columns_to_dtype:
        for n in columns:
            if n in template.columns and not all([isinstance(val, c_dtype)
                                                  for val in template[n]]):
                raise QiitaDBColumnError("The '%s' column includes values "
                                         "that cannot be cast into a %s "
                                         "value " % (n, english_desc))

    initial_columns = set(template.columns)

    if index not in template.columns:
        raise QiitaDBColumnError("The '%s' column is missing from "
                                 "your template, this file cannot be parsed."
                                 % index)

    # remove rows that have no sample identifier but that may have other data
    # in the rest of the columns
    template.dropna(subset=[index], how='all', inplace=True)

    # set the sample name as the index
    template.set_index(index, inplace=True)

    # it is not uncommon to find templates that have empty columns
    template.dropna(how='all', axis=1, inplace=True)

    initial_columns.remove(index)
    dropped_cols = initial_columns - set(template.columns)
    if dropped_cols:
        warnings.warn('The following column(s) were removed from the template '
                      'because all their values are empty: '
                      '%s' % ', '.join(dropped_cols), QiitaDBWarning)

    # Pandas represents data with np.nan rather than Nones, change it to None
    # because psycopg2 knows that a None is a Null in SQL, while it doesn't
    # know what to do with NaN
    template = template.where((pd.notnull(template)), None)

    return template