Python general.ensure_dir函数代码示例

def update_metadata(package_metadata, version=SHORTVERSION):
    """
    We've zipped the packages, and now have unzipped & zipped sizes.
    Update this info in the local metadata (but not inside the zip)
    """
    master_filepath = get_language_pack_availability_filepath(version=version)
    master_metadata = softload_json(master_filepath, logger=logging.warn, errmsg="Error opening master language pack metadata")

    for lc, updated_meta in package_metadata.iteritems():
        lang_code_ietf = lcode_to_ietf(lc)

        # Gather existing metadata
        metadata_filepath = get_language_pack_metadata_filepath(lang_code_ietf, version=version)
        stored_meta = softload_json(metadata_filepath, logger=logging.warn, errmsg="Error opening %s language pack metadata" % lc)

        stored_meta.update(updated_meta)

        # Write locally (this is used on download by distributed server to update it's database)
        with open(metadata_filepath, 'w') as output:
            json.dump(stored_meta, output)

        # Update master (this is used for central server to handle API requests for data)
        master_metadata[lang_code_ietf] = stored_meta

    # Save updated master
    ensure_dir(os.path.dirname(master_filepath))
    with open(master_filepath, 'w') as output:
        json.dump(master_metadata, output)
    logging.info("Local record of translations updated")

def unpack_zipfile_to_content_folder(zf):
    try:
        channel = zf.read("channel.name")
        folder = os.path.join(settings.ASSESSMENT_ITEM_ROOT, channel)
    except KeyError:
        # 0.16 legacy assessment zip no longer comes with a channel.name file
        folder = settings.KHAN_ASSESSMENT_ITEM_ROOT

    logging.info("Unpacking to folder {}...".format(folder))

    ensure_dir(folder)
    zf.extractall(folder)

    # If assessmentitems.version exists, copy it to another location outside
    # of the channel folder because for some reason a test expects it to be
    # there.
    version_file = os.path.join(folder, 'assessmentitems.version')
    version_file_copied_dest = os.path.join(
        settings.ASSESSMENT_ITEM_ROOT,
        'assessmentitems.version'
    )
    if version_file_copied_dest != version_file:
        if os.path.isfile(version_file_copied_dest):
            os.unlink(version_file_copied_dest)
        # Test that file exists because there's a test that mocks unzipping and
        # then this would fail because a file that should exist doesn't (doh)
        if os.path.isfile(version_file):
            # Ensure that special files are in their configured locations
            shutil.copy(
                version_file,
                version_file_copied_dest
            )

    def __init__(self, *args, **kwargs):
        """
        Force setting up live server test.  Adding to kwargs doesn't work, need to go to env.
        Dependent on how Django works here.
        """

        self.failfast = kwargs.get("failfast", False)  # overload
        # verbosity level, default 1
        self.verbosity = int(kwargs.get("verbosity"))

        # If no liveserver specified, set some default.
        #   port range is the set of open ports that Django can use to
        #   start the server.  They may have multiple servers open at once.
        if not os.environ.get('DJANGO_LIVE_TEST_SERVER_ADDRESS',""):
            os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = "localhost:9000-9999"

        self._bdd_only = kwargs["bdd_only"]  # Extra options from our custom test management command are passed into
        self._no_bdd = kwargs['no_bdd']      # the constructor, but not the build_suite function where we need them.

        # Django < 1.7 serves static files using the staticfiles app, not from static root.
        # This causes django_js_reverse not to get served to the client, so we manually copy it into distributed.

        call_command("collectstatic_js_reverse", interactive=False)

        ensure_dir(os.path.join(os.path.dirname(os.path.dirname(__file__)), "distributed", "static", "django_js_reverse", "js"))
        shutil.copy2(os.path.join(settings.STATIC_ROOT, "django_js_reverse", "js", "reverse.js"),
            os.path.join(os.path.dirname(os.path.dirname(__file__)), "distributed", "static", "django_js_reverse", "js", "reverse.js"))

        if os.environ.get("TRAVIS"):
            settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP = True

        return super(KALiteTestRunner, self).__init__(*args, **kwargs)

def move_srts(lang_code):
    """
    Srts live in the locale directory, but that's not exposed at any URL.  So instead,
    we have to move the srts out to /static/subtitles/[lang_code]/
    """
    lang_code_ietf = lcode_to_ietf(lang_code)
    lang_code_django = lcode_to_django_dir(lang_code)

    subtitles_static_dir = os.path.join(settings.USER_STATIC_FILES, "subtitles")
    src_dir = os.path.join(settings.USER_WRITABLE_LOCALE_DIR, lang_code_django, "subtitles")
    dest_dir = get_srt_path(lang_code_django)
    ensure_dir(dest_dir)

    lang_subtitles = glob.glob(os.path.join(src_dir, "*.srt"))
    logging.info("Moving %d subtitles from %s to %s" % (len(lang_subtitles), src_dir, dest_dir))

    for fil in lang_subtitles:
        srt_dest_path = os.path.join(dest_dir, os.path.basename(fil))
        if os.path.exists(srt_dest_path):
            os.remove(srt_dest_path)  # we're going to replace any srt with a newer version
        shutil.move(fil, srt_dest_path)

    if not os.path.exists(src_dir):
        logging.info("No subtitles for language pack %s" % lang_code)
    elif os.listdir(src_dir):
        logging.warn("%s is not empty; will not remove.  Please check that all subtitles were moved." % src_dir)
    else:
        logging.info("Removing empty source directory (%s)." % src_dir)
        shutil.rmtree(src_dir)

    def handle(self, *args, **options):
        if not settings.CENTRAL_SERVER:
            raise CommandError("Disabled for distributed servers, until we can figure out what to do with ")

        options['platform'] = options['platform'].lower() # normalize

        if options['platform'] not in ["all", "linux", "macos", "darwin", "windows"]:
            raise CommandError("Unrecognized platform: %s; will include ALL files." % options['platform'])

        # Step 0: refresh all resources
        get_dubbed_video_map(force=True)  # force a remote download

        # Step 1: recursively add all static files
        kalite_base = os.path.realpath(settings.PROJECT_PATH + "/../")
        files_dict = recursively_add_files(dirpath=kalite_base, **options)

        # Step 2: Add a local_settings.py file.
        #   For distributed servers, this is a copy of the local local_settings.py,
        #   with a few properties (specified as command-line options) overridden
        ls_file = create_local_settings_file(location=os.path.realpath(kalite_base+"/kalite/local_settings.py"), server_type=options['server_type'], locale=options['locale'], central_server=options["central_server"])
        files_dict[ls_file] = { "dest_path": "kalite/local_settings.py" }

        # Step 3: select output file.
        if not options['file']:
            options['file'] = create_default_archive_filename(options)

        # Step 4: package into a zip file
        ensure_dir(os.path.realpath(os.path.dirname(options["file"])))  # allows relative paths to be passed.===
        system_specific_zipping(
            files_dict = dict([(v["dest_path"], src_path) for src_path, v in files_dict.iteritems()]),
            zip_file = options["file"],
            compression=ZIP_DEFLATED if options['compress'] else ZIP_STORED,
            callback=_default_callback_zip if options["verbosity"] else None,
        )

def run_makemessages(ignore_patterns_py=[], ignore_patterns_js=[], verbosity=0):
    """Run makemessages command for english po files"""

    # Do some packages only
    python_package_dirs = glob.glob(os.path.join(PROJECT_ROOT, 'python-packages', '*'))
    ignored_packages = [os.path.join('*/python-packages/', os.path.basename(pp)) for pp in python_package_dirs if os.path.basename(pp) not in ['securesync', 'fle_utils']]

    # Besides externally requested ignores, add on a few standard ones.
    ignore_shared = ignored_packages + ['*/data/*', '*/.git/*', '*/migrations/*', '*/node_modules/*', '*/fle_utils/chronograph/*']
    ignore_patterns_py = ignore_patterns_py + ignore_shared + ['*/static-libraries/*']
    ignore_patterns_js = ignore_patterns_js + ignore_shared + ['*/kalite/static/*', '*/static-libraries/admin/*', '*/static-libraries/js/i18n/*', '*/kalite/distributed/static/khan-exercises/*'] + ['*jquery*', '*bootstrap*']

    logging.debug("Creating / validating locale root folder")
    ensure_dir(LOCALE_ROOT)

    # Command must be run from project root
    logging.debug("Moving to project root directory")
    os.chdir(PROJECT_ROOT)

    call_command('clean_pyc', path=PROJECT_ROOT)

    logging.info("Executing makemessages command")
    # Generate english po file
    sys.stdout.write("\n\nCompiling .py / .html files... ")
    call_command('makemessages', extensions=['html', 'py'], verbosity=verbosity, locale='en', ignore_patterns=ignore_patterns_py, no_obsolete=True)

    # Generate english po file for javascript
    sys.stdout.write("\n\nCompiling .js files... ")
    call_command('makemessages', extensions=['js'], domain='djangojs', verbosity=verbosity, locale='en', ignore_patterns=ignore_patterns_js, no_obsolete=True)

def update_jsi18n_file(code="en"):
    """
    For efficieny's sake, we want to cache Django's
    js18n file.  So, generate that file here, then
    save to disk--it won't change until the next language pack update!
    """
    translation.activate(code)  # we switch the language of the whole thread
    output_dir = os.path.join(settings.CONTENT_ROOT, 'locale', 'js', 'i18n')
    ensure_dir(output_dir)
    output_file = os.path.join(output_dir, "%s.js" % code)

    request = HttpRequest()
    request.path = output_file
    request.session = {settings.LANGUAGE_COOKIE_NAME: code}

    response = javascript_catalog(request, packages=('ka-lite.locale',), domain="djangojs")
    icu_js = ""
    for path in settings.LOCALE_PATHS:
        try:
            icu_js = open(os.path.join(path, code, "%s_icu.js" % code), "r").read()
        except IOError:
            logging.warn("No {code}_icu.js file found in locale_path {path}".format(code=code, path=path))
    output_js = response.content + "\n" + icu_js
    logging.info("Writing i18nized js file to {0}".format(output_file))
    with open(output_file, "w") as fp:
        fp.write(output_js)
    translation.deactivate()

    def handle(self, *args, **options):

        self.setup(options)

        operation = args[0]
        self.foreground = options.get('foreground', False)
        self.is_template = options.get('template', False)
        self.force = options.get('force', False)

        if self.is_template:
            ensure_dir(django_settings.DB_CONTENT_ITEM_TEMPLATE_DIR)
        
        # This is sort of undefined, because templates are always assumed fine
        # to overwrite
        if self.is_template and self.force:
            raise CommandError("Cannot combine --force and --template.")

        if operation == "download":
            self.start(_("Downloading content pack."))
            self.download(*args, **options)
        elif operation == "local":
            self.start(_("Installing a local content pack."))
            self.local(*args, **options)
        elif operation == "empty":
            self.empty(*args, **options)
        else:
            raise CommandError("Unknown operation: %s" % operation)

def unpack_zipfile_to_content_folder(zf):
    try:
        channel = zf.read("channel.name")
    except KeyError:
        channel = ""

    if channel:

        folder = os.path.join(settings.ASSESSMENT_ITEM_ROOT, channel)

    else:
        folder = settings.ASSESSMENT_ITEM_ROOT

    ensure_dir(folder)
    zf.extractall(folder)

    ensure_dir(settings.KHAN_ASSESSMENT_ITEM_ROOT)
    # Ensure that special files are in their configured locations
    os.rename(
        os.path.join(folder, 'assessmentitems.version'),
        settings.KHAN_ASSESSMENT_ITEM_VERSION_PATH
    )
    os.rename(
        os.path.join(folder, 'assessmentitems.sqlite'),
        settings.KHAN_ASSESSMENT_ITEM_DATABASE_PATH
    )
    # JSON file is apparrently not required (not in the test at least)
    if os.path.isfile(os.path.join(folder, 'assessmentitems.json')):
        os.rename(
            os.path.join(folder, 'assessmentitems.json'),
            settings.KHAN_ASSESSMENT_ITEM_JSON_PATH
        )

    def handle(self, *args, **options):

        # Get the CSV data, either from a recent cache_file
        #   or from the internet
        cache_dir = settings.MEDIA_ROOT
        cache_file = os.path.join(cache_dir, "dubbed_videos.csv")
        if not options["force"] and os.path.exists(cache_file) and datediff(datetime.datetime.now(), datetime.datetime.fromtimestamp(os.path.getctime(cache_file)), units="days") <= 14.0:
            # Use cached data to generate the video map
            csv_data = open(cache_file, "r").read()
            (video_map, _) = generate_dubbed_video_mappings(csv_data=csv_data)

        else:
            # Use cached data to generate the video map
            (video_map, csv_data) = generate_dubbed_video_mappings()

            try:
                ensure_dir(cache_dir)
                with open(cache_file, "w") as fp:
                    fp.write(csv_data)
            except Exception as e:
                logging.error("Failed to make a local cache of the CSV data: %s" % e)

        # Now we've built the map.  Save it.
        out_file = DUBBED_VIDEOS_MAPPING_FILEPATH
        ensure_dir(os.path.dirname(out_file))
        logging.info("Saving data to %s" % out_file)
        with open(out_file, "w") as fp:
            json.dump(video_map, fp)

        logging.info("Done.")

    def move_files(self):
        """If necessary (determined previously), move video files on disk.
        Otherwise, write into local_settings."""

        # Move over videos
        if self.move_videos == "y":
            if os.path.exists(settings.CONTENT_ROOT):
                video_files = set(glob.glob(settings.CONTENT_ROOT + '*')) - set((settings.CONTENT_ROOT + "note.txt",))
            else:
                video_files = set()
            sys.stdout.write("* Moving over %d files (videos and thumbnails)\n" % len(video_files))
            if not os.path.exists(self.working_dir + "/content/"):
                os.mkdir(self.working_dir + "/content/")
            for video_file in video_files:
                shutil.move(video_file, self.working_dir + "/content/" + os.path.split(video_file)[1])

        else:  # write (append)
            fh = open(self.working_dir + "/kalite/local_settings.py", "a")
            fh.write("\nCONTENT_ROOT = '%s'\n" % settings.CONTENT_ROOT)
            fh.close()

        # Move inner zip file
        if not os.path.exists(self.inner_zip_file) or not os.path.exists(self.signature_file):
            sys.stderr.write("\tCould not find inner zip file / signature file for storage.  Continuing...\n")
        else:
            try:
                zip_dir = os.path.join(self.working_dir, "kalite", "static", "zip")
                ensure_dir(zip_dir)
                shutil.move(self.inner_zip_file, os.path.join(zip_dir, os.path.basename(self.inner_zip_file)))
                shutil.move(self.signature_file, os.path.join(zip_dir, os.path.basename(self.signature_file)))
            except Exception as e:
                sys.stderr.write("\tCould not keep inner zip file / signature for future re-packaging (%s).  Continuing...\n" % e)

def reset_sqlite_database(username=None, email=None, password=None, router=None, verbosity="1"):
    """
    Resets the currently used sqlite database.  Creates the user if admin_username is passed.
    :param username: If present, creates a superuser with this username.
    :param email: If present, creates a superuser with this email.
    :param password: If present, creates a superuser with this password.
    :param router: The database router to use.
    :return: Returns the superuser created or None if no arguments are provided.
    """
    if not router:
        router = getattr(settings, 'SCREENSHOTS_ROUTER', 'default')
    db_engine = settings.DATABASES[router]['ENGINE']
    if db_engine == settings.SQLITE3_ENGINE:

        # make sure database path exists
        ensure_dir(settings.SCREENSHOTS_OUTPUT_PATH)

        new_io = StringIO()
        call_command("setup", interactive=False, stdout=new_io, verbosity=verbosity)
        call_command("generaterealdata", scenario_1=True, interactive=False, stdout=new_io, router=router, verbosity=verbosity)  # For coachreports pages
        if username and email and password:
            if int(verbosity) > 0:
                log.info('==> Creating superuser username==%s; email==%s ...' % (username, email,))
            call_command("createsuperuser", username=username, email=email,
                         interactive=False, stdout=new_io, router=router, verbosity=verbosity)
            admin_user = User.objects.get(username=username)
            admin_user.set_password(password)
            admin_user.save()
            return admin_user
    return None

    def setUp(self):
        self.tempdir_patch = patch.object(tempfile, "gettempdir")
        self.addCleanup(self.tempdir_patch.stop)
        self.gettempdir_method = self.tempdir_patch.start()

        # make sure we control the temp dir where temporary images are written
        self.fake_temp_dir = self.gettempdir_method.return_value = os.path.abspath("tmp/")
        ensure_dir(self.fake_temp_dir)

def update_templates():
    """Update template po files"""
    logging.info("Copying english po files to %s" % POT_PATH)

    #  post them to exposed URL
    ensure_dir(POT_PATH)
    shutil.copy(get_po_filepath(lang_code="en", filename="django.po"), os.path.join(POT_PATH, "kalite.pot"))
    shutil.copy(get_po_filepath(lang_code="en", filename="djangojs.po"), os.path.join(POT_PATH, "kalitejs.pot"))

    def __init__(self, *args, **kwargs):
        # It's not good to override __init__ for classes that inherit from TestCase
        # Since we're hackily inheriting here, we have to hackily invoke __init__
        # Perhaps better would be to decouple this class from the testing framework
        # by ditching the various mixins (they invoke TestCase methods) and just calling
        # selenium methods directly, as the mixins are a thin wrapper for that.
        # -- M.C. Gallaspy, 1/21/2015
        KALiteBrowserTestCase.__init__(self, "_fake_test")

        self.verbosity = kwargs['verbosity']

        # make sure output path exists and is empty
        if kwargs['output_dir']:
            self.output_path = os.path.join(
                os.path.realpath(os.getcwd()),
                kwargs['output_dir']
            )
        else:
            self.output_path = settings.SCREENSHOTS_OUTPUT_PATH
        ensure_dir(self.output_path)

        # make sure directory is empty from screenshot files
        png_path = os.path.join(self.output_path, "*%s" % settings.SCREENSHOTS_EXTENSION)
        pngs = glob.glob(png_path)
        if pngs and not kwargs['no_del']:
            self.logwarn("==> Deleting existing screenshots: %s ..." % png_path)
            for filename in pngs:
                os.remove(filename)

        # setup database to use and auto-create admin user
        self.loginfo("==> Setting-up database ...")
        self.admin_user = reset_sqlite_database(self.admin_username, self.admin_email, self.default_password, verbosity=self.verbosity)
        self.admin_pass = self.default_password
        if not self.admin_user:
            raise Exception("==> Did not successfully setup database!")

        Facility.initialize_default_facility("Facility Dos")  # Default facility required to avoid pernicious facility selection page
        facility = self.facility = Facility.objects.get(name="Facility Dos")
        self.create_student(username=self.student_username, password=self.default_password, facility=facility)
        self.create_teacher(username=self.coach_username, password=self.default_password, facility=facility)

        self.persistent_browser = True
        self.max_wait_time = kwargs.get('max_wait_time', 30)

        self.setUpClass()

        self.loginfo("==> Setting-up browser ...")
        super(Screenshot, self).setUp()
        # Selenium won't scroll to an element, so we have to make the window size is large enough so that everything is visible
        self.browser.set_window_size(1024, 768)
        # self.browser.implicitly_wait(3)

        # After initializing the server (with setUp) and a browser, set the language
        self.set_session_language(kwargs['language'])

        self.loginfo("==> Browser %s successfully setup with live_server_url %s." %
                 (self.browser.name, self.live_server_url,))
        self.loginfo("==> Saving screenshots to %s ..." % (settings.SCREENSHOTS_OUTPUT_PATH,))