EDU-894 Save all course content in course.json

EDU-891 Drop courses cache

First iteration

python/educational-core/resources/META-INF/plugin.xml

@@ -194,7 +194,6 @@
     <checkListener implementation="com.jetbrains.edu.learning.twitter.StudyTwitterAction"/>
 
     <!--course creator-->
-    <checkListener implementation="com.jetbrains.edu.coursecreator.CCCheckListener"/>
     <optionsProvider instance="com.jetbrains.edu.coursecreator.settings.CCOptions"/>
   </extensions>
 </idea-plugin>
\ No newline at end of file

python/educational-core/src/com/jetbrains/edu/coursecreator/CCCheckListener.java

-package com.jetbrains.edu.coursecreator;
-
-import com.intellij.openapi.project.Project;
-import com.intellij.openapi.vfs.VirtualFile;
-import com.jetbrains.edu.learning.StudyCheckListener;
-import com.jetbrains.edu.learning.courseFormat.tasks.Task;
-import org.jetbrains.annotations.NotNull;
-
-public class CCCheckListener implements StudyCheckListener {
-
-  @Override
-  public void beforeCheck(@NotNull Project project, @NotNull Task task) {
-    if (!CCUtils.isCourseCreator(project)) {
-      return;
-    }
-    VirtualFile taskDir = task.getTaskDir(project);
-    if (taskDir == null) {
-      return;
-    }
-    CCUtils.updateResources(project, task, taskDir);
-  }
-}

python/educational-core/src/com/jetbrains/edu/coursecreator/CCProjectComponent.java

 package com.jetbrains.edu.coursecreator;
 
 import com.intellij.openapi.application.ApplicationManager;
-import com.intellij.openapi.application.PathManager;
 import com.intellij.openapi.components.AbstractProjectComponent;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.project.Project;
@@ -15,12 +14,11 @@ import com.jetbrains.edu.learning.StudyUtils;
 import com.jetbrains.edu.learning.core.EduNames;
 import com.jetbrains.edu.learning.courseFormat.Course;
 import com.jetbrains.edu.learning.courseFormat.Lesson;
-import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 import com.jetbrains.edu.learning.courseFormat.TaskFile;
+import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 import com.jetbrains.edu.learning.statistics.EduUsagesCollector;
 import org.jetbrains.annotations.NotNull;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -47,9 +45,6 @@ public class CCProjectComponent extends AbstractProjectComponent {
       CCProjectService.getInstance(myProject).setCourse(null);
       oldCourse.initCourse(true);
       oldCourse.setCourseMode(CCUtils.COURSE_MODE);
-      File coursesDir = new File(PathManager.getConfigPath(), "courses");
-      File courseDir = new File(coursesDir, oldCourse.getName() + "-" + myProject.getName());
-      oldCourse.setCourseDirectory(courseDir.getPath());
       StudyUtils.registerStudyToolWindow(oldCourse, myProject);
       transformFiles(oldCourse, myProject);
     }

python/educational-core/src/com/jetbrains/edu/coursecreator/CCRefactoringElementListenerProvider.java

@@ -15,10 +15,7 @@
  */
 package com.jetbrains.edu.coursecreator;
 
-import com.intellij.openapi.application.ApplicationManager;
-import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.project.Project;
-import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.PsiDirectory;
 import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiFile;
@@ -30,16 +27,14 @@ import com.jetbrains.edu.learning.StudyUtils;
 import com.jetbrains.edu.learning.core.EduNames;
 import com.jetbrains.edu.learning.courseFormat.Course;
 import com.jetbrains.edu.learning.courseFormat.Lesson;
-import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 import com.jetbrains.edu.learning.courseFormat.TaskFile;
+import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.io.IOException;
 import java.util.Map;
 
 public class CCRefactoringElementListenerProvider implements RefactoringElementListenerProvider {
-  private static final Logger LOG = Logger.getInstance(CCRefactoringElementListenerProvider.class);
 
   @Nullable
   @Override
@@ -94,21 +89,8 @@ public class CCRefactoringElementListenerProvider implements RefactoringElementL
       if (taskFile == null) {
         return;
       }
-      ApplicationManager.getApplication().runWriteAction(() -> {
-        VirtualFile patternFile = StudyUtils.getPatternFile(taskFile, oldName);
-        if (patternFile != null) {
-          try {
-            patternFile.delete(CCRefactoringElementListenerProvider.class);
-          }
-          catch (IOException e) {
-            LOG.info(e);
-          }
-        }
-      });
-
       taskFiles.remove(oldName);
       taskFiles.put(StudyUtils.pathRelativeToTask(file.getVirtualFile()), taskFile);
-      CCUtils.createResourceFile(file.getVirtualFile(), course, taskDir.getVirtualFile());
     }
 
     @Override

python/educational-core/src/com/jetbrains/edu/coursecreator/CCUtils.java

@@ -8,34 +8,37 @@ import com.intellij.openapi.actionSystem.Presentation;
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.module.Module;
-import com.intellij.openapi.project.DumbModePermission;
-import com.intellij.openapi.project.DumbService;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.roots.ModuleRootModificationUtil;
 import com.intellij.openapi.roots.ProjectRootManager;
 import com.intellij.openapi.util.Ref;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.io.FileUtilRt;
-import com.intellij.openapi.vfs.LocalFileSystem;
+import com.intellij.openapi.vfs.VfsUtilCore;
 import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.openapi.vfs.VirtualFileVisitor;
 import com.intellij.psi.PsiDirectory;
 import com.intellij.util.Function;
+import com.intellij.util.containers.hash.HashMap;
 import com.jetbrains.edu.learning.EduPluginConfigurator;
 import com.jetbrains.edu.learning.StudyTaskManager;
 import com.jetbrains.edu.learning.StudyUtils;
 import com.jetbrains.edu.learning.core.EduNames;
 import com.jetbrains.edu.learning.core.EduUtils;
 import com.jetbrains.edu.learning.courseFormat.Course;
+import com.jetbrains.edu.learning.courseFormat.Lesson;
 import com.jetbrains.edu.learning.courseFormat.StudyItem;
 import com.jetbrains.edu.learning.courseFormat.TaskFile;
 import com.jetbrains.edu.learning.courseFormat.tasks.Task;
-import com.jetbrains.edu.learning.courseFormat.tasks.TaskWithSubtasks;
+import org.apache.commons.codec.binary.Base64;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.io.File;
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
 
 public class CCUtils {
   public static final String ANSWER_EXTENSION_DOTTED = ".answer.";
@@ -137,27 +140,22 @@ public class CCUtils {
       return folder;
     }
     final Ref<VirtualFile> generatedRoot = new Ref<>();
-    DumbService.allowStartingDumbModeInside(DumbModePermission.MAY_START_BACKGROUND, new Runnable() {
+    ApplicationManager.getApplication().runWriteAction(new Runnable() {
       @Override
       public void run() {
-        ApplicationManager.getApplication().runWriteAction(new Runnable() {
-          @Override
-          public void run() {
-            try {
-              generatedRoot.set(baseDir.createChildDirectory(this, GENERATED_FILES_FOLDER));
-              VirtualFile contentRootForFile =
-                ProjectRootManager.getInstance(module.getProject()).getFileIndex().getContentRootForFile(generatedRoot.get());
-              if (contentRootForFile == null) {
-                return;
-              }
-              ModuleRootModificationUtil.updateExcludedFolders(module, contentRootForFile, Collections.emptyList(),
-                                                               Collections.singletonList(generatedRoot.get().getUrl()));
-            }
-            catch (IOException e) {
-              LOG.info("Failed to create folder for generated files", e);
-            }
+        try {
+          generatedRoot.set(baseDir.createChildDirectory(this, GENERATED_FILES_FOLDER));
+          VirtualFile contentRootForFile =
+            ProjectRootManager.getInstance(module.getProject()).getFileIndex().getContentRootForFile(generatedRoot.get());
+          if (contentRootForFile == null) {
+            return;
           }
-        });
+          ModuleRootModificationUtil.updateExcludedFolders(module, contentRootForFile, Collections.emptyList(),
+                                                           Collections.singletonList(generatedRoot.get().getUrl()));
+        }
+        catch (IOException e) {
+          LOG.info("Failed to create folder for generated files", e);
+        }
       }
     });
     return generatedRoot.get();
@@ -211,63 +209,6 @@ public class CCUtils {
     return configurator.isTestFile(file);
   }
 
-  public static void createResourceFile(VirtualFile createdFile, Course course, VirtualFile taskVF) {
-    VirtualFile lessonVF = taskVF.getParent();
-    if (lessonVF == null) {
-      return;
-    }
-
-    String taskResourcesPath = FileUtil.join(course.getCourseDirectory(), lessonVF.getName(), taskVF.getName());
-    File taskResourceFile = new File(taskResourcesPath);
-    if (!taskResourceFile.exists()) {
-      if (!taskResourceFile.mkdirs()) {
-        LOG.info("Failed to create resources for task " + taskResourcesPath);
-      }
-    }
-    try {
-      File toFile = new File(taskResourceFile, createdFile.getName());
-      FileUtil.copy(new File(createdFile.getPath()), toFile);
-    }
-    catch (IOException e) {
-      LOG.info("Failed to copy created task file to resources " + createdFile.getPath());
-    }
-  }
-
-
-  public static void updateResources(Project project, Task task, VirtualFile taskDir) {
-    Course course = StudyTaskManager.getInstance(project).getCourse();
-    if (course == null) {
-      return;
-    }
-    VirtualFile lessonVF = taskDir.getParent();
-    if (lessonVF == null) {
-      return;
-    }
-
-    String taskResourcesPath = FileUtil.join(course.getCourseDirectory(), lessonVF.getName(), taskDir.getName());
-    File taskResourceFile = new File(taskResourcesPath);
-    if (!taskResourceFile.exists()) {
-      if (!taskResourceFile.mkdirs()) {
-        LOG.info("Failed to create resources for task " + taskResourcesPath);
-      }
-    }
-    VirtualFile studentDir = LocalFileSystem.getInstance().findFileByIoFile(taskResourceFile);
-    if (studentDir == null) {
-      return;
-    }
-    for (Map.Entry<String, TaskFile> entry : task.getTaskFiles().entrySet()) {
-      String name = entry.getKey();
-      VirtualFile answerFile = taskDir.findFileByRelativePath(name);
-      if (answerFile == null) {
-        continue;
-      }
-      ApplicationManager.getApplication().runWriteAction(() -> {
-        EduUtils.createStudentFile(CCUtils.class, project, answerFile, studentDir, null,
-                                   task instanceof TaskWithSubtasks ? ((TaskWithSubtasks)task).getActiveSubtaskIndex() : 0);
-      });
-    }
-  }
-
   public static void updateActionGroup(AnActionEvent e) {
     Presentation presentation = e.getPresentation();
     Project project = e.getProject();
@@ -315,4 +256,58 @@ public class CCUtils {
       }
     });
   }
+
+  @Nullable
+  public static Lesson createAdditionalLesson(Course course, Project project) {
+    final VirtualFile baseDir = project.getBaseDir();
+    EduPluginConfigurator configurator = EduPluginConfigurator.INSTANCE.forLanguage(course.getLanguageById());
+
+    final Lesson lesson = new Lesson();
+    lesson.setName(EduNames.PYCHARM_ADDITIONAL);
+    final Task task = new Task();
+    task.setLesson(lesson);
+    task.setName(EduNames.PYCHARM_ADDITIONAL);
+    task.setIndex(1);
+
+    VfsUtilCore.visitChildrenRecursively(baseDir, new VirtualFileVisitor(VirtualFileVisitor.NO_FOLLOW_SYMLINKS) {
+      @Override
+      public boolean visitFile(@NotNull VirtualFile file) {
+        final String name = file.getName();
+        if (name.equals(EduNames.COURSE_META_FILE) || name.equals(EduNames.HINTS) || name.startsWith(".")) return false;
+        String sanitizedName = FileUtil.sanitizeFileName(course.getName());
+        final String archiveName = sanitizedName.startsWith("_") ? EduNames.COURSE : sanitizedName;
+        if (name.equals(archiveName + ".zip")) return false;
+        if (GENERATED_FILES_FOLDER.equals(name) || Project.DIRECTORY_STORE_FOLDER.equals(name)) {
+          return false;
+        }
+        if (file.isDirectory()) return true;
+
+        if (StudyUtils.isTaskDescriptionFile(name) || StudyUtils.isTestsFile(project, name)) return true;
+
+        if (name.contains(".iml") || (configurator != null && configurator.excludeFromArchive(name))) {
+          return false;
+        }
+        final TaskFile taskFile = StudyUtils.getTaskFile(project, file);
+        if (taskFile == null) {
+          final String path = VfsUtilCore.getRelativePath(file, baseDir);
+          try {
+            if (EduUtils.isImage(file.getName())) {
+              task.addTestsTexts(path, Base64.encodeBase64URLSafeString(FileUtil.loadBytes(file.getInputStream())));
+            }
+            else {
+              task.addTestsTexts(path, FileUtil.loadTextAndClose(file.getInputStream()));
+            }
+          }
+          catch (IOException e) {
+            LOG.error("Can't find file " + path);
+          }
+        }
+        return true;
+      }
+    });
+    if (task.getTestsText().isEmpty()) return null;
+    lesson.addTask(task);
+    lesson.setIndex(1);
+    return lesson;
+  }
 }

python/educational-core/src/com/jetbrains/edu/coursecreator/CCVirtualFileListener.java

@@ -12,12 +12,10 @@ import com.jetbrains.edu.learning.StudyUtils;
 import com.jetbrains.edu.learning.core.EduNames;
 import com.jetbrains.edu.learning.courseFormat.Course;
 import com.jetbrains.edu.learning.courseFormat.Lesson;
-import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 import com.jetbrains.edu.learning.courseFormat.TaskFile;
+import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 import org.jetbrains.annotations.NotNull;
 
-import java.io.File;
-
 public class CCVirtualFileListener extends VirtualFileAdapter {
 
   @Override
@@ -48,7 +46,7 @@ public class CCVirtualFileListener extends VirtualFileAdapter {
     String taskRelativePath = StudyUtils.pathRelativeToTask(createdFile);
 
     EduPluginConfigurator configurator = EduPluginConfigurator.INSTANCE.forLanguage(course.getLanguageById());
-    if (configurator != null && configurator.excludeFromArchive(new File(createdFile.getPath()))) {
+    if (configurator != null && configurator.excludeFromArchive(createdFile.getName())) {
       return;
     }
 
@@ -68,8 +66,6 @@ public class CCVirtualFileListener extends VirtualFileAdapter {
       return;
     }
 
-    CCUtils.createResourceFile(createdFile, course, taskVF);
-
     task.addTaskFile(taskRelativePath, 1);
   }
 
@@ -89,7 +85,7 @@ public class CCVirtualFileListener extends VirtualFileAdapter {
       return;
     }
     Course course = StudyTaskManager.getInstance(project).getCourse();
-    if (course == null || path.contains(FileUtil.toSystemIndependentName(course.getCourseDirectory()))) {
+    if (course == null) {
       return;
     }
     final TaskFile taskFile = StudyUtils.getTaskFile(project, removedFile);
@@ -112,7 +108,7 @@ public class CCVirtualFileListener extends VirtualFileAdapter {
     }
     VirtualFile courseDir = project.getBaseDir();
     CCUtils.updateHigherElements(courseDir.getChildren(), file -> course.getLesson(file.getName()), removedLesson.getIndex(), EduNames.LESSON, -1);
-    course.getLessons().remove(removedLesson);
+    course.removeLesson(removedLesson);
   }
 
   private static void deleteTask(@NotNull final Course course, @NotNull final VirtualFile removedTask) {

python/educational-core/src/com/jetbrains/edu/coursecreator/actions/CCAddAsTaskFile.java

@@ -9,8 +9,8 @@ import com.jetbrains.edu.coursecreator.CCUtils;
 import com.jetbrains.edu.learning.StudyUtils;
 import com.jetbrains.edu.learning.core.EduUtils;
 import com.jetbrains.edu.learning.courseFormat.Course;
-import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 import com.jetbrains.edu.learning.courseFormat.TaskFile;
+import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 
 public class CCAddAsTaskFile extends CCTaskFileActionBase {
   public static final String ACTION_NAME = "Make Visible to Student";
@@ -21,7 +21,7 @@ public class CCAddAsTaskFile extends CCTaskFileActionBase {
 
 
   protected void performAction(VirtualFile file, Task task, Course course, Project project) {
-    EduUtils.runUndoableAction(project, ACTION_NAME, new AddTaskFile(file, null, course, project, task));
+    EduUtils.runUndoableAction(project, ACTION_NAME, new AddTaskFile(file, null, project, task));
   }
 
   protected boolean isAvailable(Project project, VirtualFile file) {
@@ -31,21 +31,20 @@ public class CCAddAsTaskFile extends CCTaskFileActionBase {
   private static class AddTaskFile extends BasicUndoableAction {
     private final VirtualFile myFile;
     private TaskFile myTaskFile;
-    private final Course myCourse;
     private final Project myProject;
     private final Task myTask;
 
-    public AddTaskFile(VirtualFile file, TaskFile taskFile, Course course, Project project, Task task) {
+    public AddTaskFile(VirtualFile file, TaskFile taskFile, Project project, Task task) {
       super(file);
       myFile = file;
       myTaskFile = taskFile;
-      myCourse = course;
       myProject = project;
       myTask = task;
     }
 
     @Override
     public void undo() throws UnexpectedUndoException {
+      if (myTaskFile == null) return;
       CCHideFromStudent.hideFromStudent(myFile, myProject, myTask.getTaskFiles(), myTaskFile);
       ProjectView.getInstance(myProject).refresh();
     }
@@ -59,7 +58,6 @@ public class CCAddAsTaskFile extends CCTaskFileActionBase {
         myTask.addTaskFile(taskRelativePath, myTask.getTaskFiles().size());
         myTaskFile = myTask.getTaskFile(taskRelativePath);
       }
-      CCUtils.createResourceFile(myFile, myCourse, StudyUtils.getTaskDir(myFile));
       ProjectView.getInstance(myProject).refresh();
     }
 

python/educational-core/src/com/jetbrains/edu/coursecreator/actions/CCCreateCourseArchive.java

@@ -14,30 +14,31 @@ import com.intellij.openapi.project.DumbAwareAction;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.ui.DialogWrapper;
 import com.intellij.openapi.ui.Messages;
-import com.intellij.openapi.util.Condition;
-import com.intellij.openapi.util.io.FileUtil;
-import com.intellij.openapi.vfs.VfsUtil;
+import com.intellij.openapi.vfs.VfsUtilCore;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.VirtualFileManager;
-import com.intellij.util.containers.ContainerUtil;
+import com.intellij.util.containers.hash.HashMap;
 import com.intellij.util.io.ZipUtil;
 import com.jetbrains.edu.coursecreator.CCUtils;
 import com.jetbrains.edu.coursecreator.ui.CreateCourseArchiveDialog;
 import com.jetbrains.edu.learning.StudySerializationUtils;
-import com.jetbrains.edu.learning.EduPluginConfigurator;
 import com.jetbrains.edu.learning.StudyTaskManager;
+import com.jetbrains.edu.learning.StudyUtils;
 import com.jetbrains.edu.learning.core.EduNames;
 import com.jetbrains.edu.learning.core.EduUtils;
 import com.jetbrains.edu.learning.courseFormat.Course;
 import com.jetbrains.edu.learning.courseFormat.Lesson;
+import com.jetbrains.edu.learning.courseFormat.TaskFile;
 import com.jetbrains.edu.learning.courseFormat.tasks.Task;
-import com.jetbrains.edu.learning.courseFormat.tasks.TaskWithSubtasks;
 import com.jetbrains.edu.learning.statistics.EduUsagesCollector;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.*;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 import java.util.zip.ZipOutputStream;
 
 public class CCCreateCourseArchive extends DumbAwareAction {
@@ -90,34 +91,26 @@ public class CCCreateCourseArchive extends DumbAwareAction {
       return;
     }
 
-    EduPluginConfigurator configurator = EduPluginConfigurator.INSTANCE.forLanguage(course.getLanguageById());
-    if (configurator == null) {
-      return;
-    }
-    FileFilter filter = pathname -> !configurator.excludeFromArchive(pathname);
-
-    for (VirtualFile child : baseDir.getChildren()) {
-      String name = child.getName();
-      File fromFile = new File(child.getPath());
-      if (CCUtils.GENERATED_FILES_FOLDER.equals(name) || Project.DIRECTORY_STORE_FOLDER.equals(name)
-          || name.contains("iml") || configurator.excludeFromArchive(fromFile)) {
-        continue;
-      }
-      copyChild(archiveFolder, filter, child, fromFile);
-    }
-
     ApplicationManager.getApplication().runWriteAction(new Runnable() {
       @Override
       public void run() {
         archiveFolder.refresh(false, true);
         Course courseCopy = course.copy();
         replaceAnswerFilesWithTaskFiles(courseCopy);
+        additionalFiles(courseCopy);
         generateJson(archiveFolder, courseCopy);
         VirtualFileManager.getInstance().refreshWithoutFileWatcher(false);
         packCourse(archiveFolder, locationDir, zipName, showMessage);
         synchronize(project);
       }
 
+      private void additionalFiles(Course course) {
+        final Lesson lesson = CCUtils.createAdditionalLesson(course, project);
+        if (lesson != null) {
+          course.addLesson(lesson);
+        }
+      }
+
       private void replaceAnswerFilesWithTaskFiles(Course courseCopy) {
         for (Lesson lesson : courseCopy.getLessons()) {
           String lessonDirName = EduNames.LESSON + String.valueOf(lesson.getIndex());
@@ -126,61 +119,76 @@ public class CCCreateCourseArchive extends DumbAwareAction {
           for (Task task : lesson.getTaskList()) {
             final VirtualFile taskDir = task.getTaskDir(project);
             if (taskDir == null) continue;
-            String taskDirName = EduNames.TASK + String.valueOf(task.getIndex());
-            VirtualFile studentFileDir = VfsUtil.findRelativeFile(archiveFolder, lessonDirName, taskDirName);
-            if (studentFileDir == null) {
-              continue;
-            }
-            VirtualFile srcDir = studentFileDir.findChild(EduNames.SRC);
-            if (srcDir != null) {
-              studentFileDir = srcDir;
-            }
-            if (task instanceof TaskWithSubtasks) {
-              transformSubtaskTestsToTextFiles(studentFileDir);
-            }
-            for (String taskFile : task.getTaskFiles().keySet()) {
-              VirtualFile answerFile = taskDir.findFileByRelativePath(taskFile);
-              if (answerFile == null) {
-                continue;
-              }
-              EduUtils.createStudentFile(this, project, answerFile, studentFileDir, task, 0);
-            }
+            convertToStudentTaskFiles(task, taskDir);
+            addTestsToTask(task);
+            addDescriptions(task);
           }
         }
       }
 
-      private void transformSubtaskTestsToTextFiles(VirtualFile studentFileDir) {
-        Condition<VirtualFile> isSubtaskTestFile =
-          file -> CCUtils.isTestsFile(project, file) && file.getName().contains(EduNames.SUBTASK_MARKER);
-        List<VirtualFile> subtaskTests = ContainerUtil.filter(Arrays.asList(studentFileDir.getChildren()), isSubtaskTestFile);
-        for (VirtualFile subtaskTest : subtaskTests) {
+      private void convertToStudentTaskFiles(Task task, VirtualFile taskDir) {
+        final HashMap<String, TaskFile> studentTaskFiles = new HashMap<>();
+        for (Map.Entry<String, TaskFile> entry : task.getTaskFiles().entrySet()) {
+          VirtualFile answerFile = taskDir.findFileByRelativePath(entry.getKey());
+          if (answerFile == null) {
+            continue;
+          }
+          final TaskFile studentFile = EduUtils.createStudentFile(project, answerFile, task, 0);
+          if (studentFile != null) {
+            studentTaskFiles.put(entry.getKey(), studentFile);
+          }
+        }
+        task.taskFiles = studentTaskFiles;
+      }
+
+      private void addDescriptions(@NotNull final Task task) {
+        final List<VirtualFile> descriptions = getDescriptionFiles(task, project);
+        for (VirtualFile file : descriptions) {
           try {
-            subtaskTest.rename(this, subtaskTest.getNameWithoutExtension() + ".txt");
+            task.addTaskText(file.getName(), VfsUtilCore.loadText(file));
           }
           catch (IOException e) {
-            LOG.error(e);
+            LOG.warn("Failed to load text " + file.getName());
           }
         }
       }
-    });
-  }
 
-  private static void copyChild(VirtualFile archiveFolder, FileFilter filter, VirtualFile child, File fromFile) {
-    File toFile = new File(archiveFolder.getPath(), child.getName());
+      private void addTestsToTask(Task task) {
+        final List<VirtualFile> testFiles = getTestFiles(task, project);
+        for (VirtualFile file : testFiles) {
+          try {
+            task.addTestsTexts(file.getName(), VfsUtilCore.loadText(file));
+          }
+          catch (IOException e) {
+            LOG.warn("Failed to load text " + file.getName());
+          }
+        }
+      }
 
-    try {
-      if (child.isDirectory()) {
-        FileUtil.copyDir(fromFile, toFile, filter);
+      private List<VirtualFile> getTestFiles(@NotNull Task task, @NotNull Project project) {
+        List<VirtualFile> testFiles = new ArrayList<>();
+        VirtualFile taskDir = task.getTaskDir(project);
+        if (taskDir == null) {
+          return testFiles;
+        }
+        testFiles.addAll(Arrays.stream(taskDir.getChildren())
+                           .filter(file -> StudyUtils.isTestsFile(project, file.getName()))
+                           .collect(Collectors.toList()));
+        return testFiles;
       }
-      else {
-        if (filter.accept(fromFile)) {
-          FileUtil.copy(fromFile, toFile);
+
+      private List<VirtualFile> getDescriptionFiles(@NotNull Task task, @NotNull Project project) {
+        List<VirtualFile> testFiles = new ArrayList<>();
+        VirtualFile taskDir = task.getTaskDir(project);
+        if (taskDir == null) {
+          return testFiles;
         }
+        testFiles.addAll(Arrays.stream(taskDir.getChildren())
+                           .filter(file -> StudyUtils.isTaskDescriptionFile(file.getName()))
+                           .collect(Collectors.toList()));
+        return testFiles;
       }
-    }
-    catch (IOException e) {
-      LOG.info("Failed to copy" + fromFile.getPath(), e);
-    }
+    });
   }
 
   private static void synchronize(@NotNull final Project project) {

python/educational-core/src/com/jetbrains/edu/coursecreator/actions/CCFromCourseArchive.java

@@ -27,7 +27,10 @@ import com.jetbrains.edu.learning.StudySerializationUtils;
 import com.jetbrains.edu.learning.StudyTaskManager;
 import com.jetbrains.edu.learning.core.EduDocumentListener;
 import com.jetbrains.edu.learning.core.EduNames;
-import com.jetbrains.edu.learning.courseFormat.*;
+import com.jetbrains.edu.learning.courseFormat.AnswerPlaceholder;
+import com.jetbrains.edu.learning.courseFormat.Course;
+import com.jetbrains.edu.learning.courseFormat.Lesson;
+import com.jetbrains.edu.learning.courseFormat.TaskFile;
 import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 import org.jetbrains.annotations.NotNull;
 
@@ -35,8 +38,6 @@ import java.io.*;
 import java.util.ArrayList;
 import java.util.Map;
 
-import static com.jetbrains.edu.learning.courseGeneration.StudyProjectGenerator.OUR_COURSES_DIR;
-
 public class CCFromCourseArchive extends DumbAwareAction {
   private static final Logger LOG = Logger.getInstance(CCFromCourseArchive.class.getName());
 
@@ -80,8 +81,6 @@ public class CCFromCourseArchive extends DumbAwareAction {
       }
 
       StudyTaskManager.getInstance(project).setCourse(course);
-      File courseDir = new File(OUR_COURSES_DIR, course.getName() + "-" + project.getName());
-      course.setCourseDirectory(courseDir.getPath());
       course.setCourseMode(CCUtils.COURSE_MODE);
       project.getBaseDir().refresh(false, true);
       int index = 1;

python/educational-core/src/com/jetbrains/edu/coursecreator/actions/CCGetCourseFromStepic.java

@@ -15,25 +15,22 @@ import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.VirtualFileManager;
 import com.jetbrains.edu.coursecreator.CCUtils;
+import com.jetbrains.edu.coursecreator.stepik.CCStepicConnector;
 import com.jetbrains.edu.learning.StudyTaskManager;
 import com.jetbrains.edu.learning.StudyUtils;
 import com.jetbrains.edu.learning.core.EduNames;
 import com.jetbrains.edu.learning.courseFormat.Course;
+import com.jetbrains.edu.learning.courseFormat.CourseInfo;
 import com.jetbrains.edu.learning.courseFormat.Lesson;
-import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 import com.jetbrains.edu.learning.courseFormat.TaskFile;
+import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 import com.jetbrains.edu.learning.courseGeneration.StudyGenerator;
-import com.jetbrains.edu.coursecreator.stepik.CCStepicConnector;
-import com.jetbrains.edu.learning.courseFormat.CourseInfo;
 import com.jetbrains.edu.learning.stepic.EduStepicConnector;
 import org.jetbrains.annotations.NotNull;
 
-import java.io.File;
 import java.util.Map;
 
 import static com.jetbrains.edu.coursecreator.actions.CCFromCourseArchive.createAnswerFile;
-import static com.jetbrains.edu.learning.courseGeneration.StudyProjectGenerator.OUR_COURSES_DIR;
-import static com.jetbrains.edu.learning.courseGeneration.StudyProjectGenerator.flushCourse;
 
 public class CCGetCourseFromStepic extends DumbAwareAction {
 
@@ -66,21 +63,17 @@ public class CCGetCourseFromStepic extends DumbAwareAction {
 
     final Course course = EduStepicConnector.getCourse(project, info);
     if (course != null) {
-      flushCourse(course);
 
-      final File courseDirectory = StudyUtils.getCourseDirectory(course);
       ApplicationManager.getApplication().invokeAndWait(() -> ApplicationManager.getApplication().runWriteAction(() -> {
         final VirtualFile[] children = baseDir.getChildren();
         for (VirtualFile child : children) {
           StudyUtils.deleteFile(child);
         }
-        StudyGenerator.createCourse(course, baseDir, courseDirectory, project);
+        StudyGenerator.createCourse(course, baseDir);
       }));
 
 
       StudyTaskManager.getInstance(project).setCourse(course);
-      File courseDir = new File(OUR_COURSES_DIR, course.getName() + "-" + project.getName());
-      course.setCourseDirectory(courseDir.getPath());
       course.setCourseMode(CCUtils.COURSE_MODE);
       project.getBaseDir().refresh(false, true);
       int index = 1;

python/educational-core/src/com/jetbrains/edu/coursecreator/actions/CCHideFromStudent.java

 package com.jetbrains.edu.coursecreator.actions;
 
 import com.intellij.ide.projectView.ProjectView;
-import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.command.undo.BasicUndoableAction;
 import com.intellij.openapi.command.undo.UnexpectedUndoException;
-import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.editor.Editor;
 import com.intellij.openapi.fileEditor.FileEditor;
 import com.intellij.openapi.fileEditor.FileEditorManager;
 import com.intellij.openapi.fileEditor.TextEditor;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.vfs.VirtualFile;
-import com.jetbrains.edu.coursecreator.CCUtils;
 import com.jetbrains.edu.learning.StudyUtils;
 import com.jetbrains.edu.learning.core.EduUtils;
 import com.jetbrains.edu.learning.courseFormat.Course;
-import com.jetbrains.edu.learning.courseFormat.tasks.Task;
 import com.jetbrains.edu.learning.courseFormat.TaskFile;
+import com.jetbrains.edu.learning.courseFormat.tasks.Task;
+import org.jetbrains.annotations.NotNull;
 
-import java.io.IOException;
 import java.util.Map;
 
 public class CCHideFromStudent extends CCTaskFileActionBase {
-
-  private static final Logger LOG = Logger.getInstance(CCHideFromStudent.class);
   private static final String ACTION_NAME = "Hide from Student";
 
   public CCHideFromStudent() {
@@ -36,21 +31,19 @@ public class CCHideFromStudent extends CCTaskFileActionBase {
     if (taskFile == null) {
       return;
     }
-    EduUtils.runUndoableAction(project, ACTION_NAME, new HideTaskFile(project, course, file, task, taskFile));
+    EduUtils.runUndoableAction(project, ACTION_NAME, new HideTaskFile(project, file, task, taskFile));
   }
 
   private static class HideTaskFile extends BasicUndoableAction {
 
     private final Project myProject;
-    private final Course myCourse;
     private final VirtualFile myFile;
     private final Task myTask;
     private final TaskFile myTaskFile;
 
-    public HideTaskFile(Project project, Course course, VirtualFile file, Task task, TaskFile taskFile) {
+    public HideTaskFile(Project project, VirtualFile file, Task task, TaskFile taskFile) {
       super(file);
       myProject = project;
-      myCourse = course;
       myFile = file;
       myTask = task;
       myTaskFile = taskFile;
@@ -59,7 +52,6 @@ public class CCHideFromStudent extends CCTaskFileActionBase {
     @Override
     public void undo() throws UnexpectedUndoException {
       myTask.getTaskFiles().put(StudyUtils.pathRelativeToTask(myFile), myTaskFile);
-      CCUtils.createResourceFile(myFile, myCourse, StudyUtils.getTaskDir(myFile));
       if (!myTaskFile.getAnswerPlaceholders().isEmpty() && FileEditorManager.getInstance(myProject).isFileOpen(myFile)) {
         for (FileEditor fileEditor : FileEditorManager.getInstance(myProject).getEditors(myFile)) {
           if (fileEditor instanceof TextEditor) {
@@ -83,7 +75,7 @@ public class CCHideFromStudent extends CCTaskFileActionBase {
     }
   }
 
-  public static void hideFromStudent(VirtualFile file, Project project, Map<String, TaskFile> taskFiles, TaskFile taskFile) {
+  public static void hideFromStudent(VirtualFile file, Project project, Map<String, TaskFile> taskFiles, @NotNull final TaskFile taskFile) {
     if (!taskFile.getAnswerPlaceholders().isEmpty() && FileEditorManager.getInstance(project).isFileOpen(file)) {
       for (FileEditor fileEditor : FileEditorManager.getInstance(project).getEditors(file)) {
         if (fileEditor instanceof TextEditor) {
@@ -93,17 +85,6 @@ public class CCHideFromStudent extends CCTaskFileActionBase {
       }
     }
     String taskRelativePath = StudyUtils.pathRelativeToTask(file);
-    VirtualFile patternFile = StudyUtils.getPatternFile(taskFile, taskRelativePath);
-    ApplicationManager.getApplication().runWriteAction(() -> {
-      if (patternFile != null) {
-        try {
-          patternFile.delete(CCHideFromStudent.class);
-        }
-        catch (IOException e) {
-          LOG.info(e);
-        }
-      }
-    });
     taskFiles.remove(taskRelativePath);
   }
 

python/educational-core/src/com/jetbrains/edu/coursecreator/actions/CCPushCourse.java

@@ -11,12 +11,12 @@ import com.intellij.openapi.progress.Task;
 import com.intellij.openapi.project.DumbAwareAction;
 import com.intellij.openapi.project.Project;
 import com.jetbrains.edu.coursecreator.CCUtils;
+import com.jetbrains.edu.coursecreator.stepik.CCStepicConnector;
 import com.jetbrains.edu.learning.StudyTaskManager;
 import com.jetbrains.edu.learning.courseFormat.Course;
+import com.jetbrains.edu.learning.courseFormat.CourseInfo;
 import com.jetbrains.edu.learning.courseFormat.Lesson;
 import com.jetbrains.edu.learning.statistics.EduUsagesCollector;
-import com.jetbrains.edu.coursecreator.stepik.CCStepicConnector;
-import com.jetbrains.edu.learning.courseFormat.CourseInfo;
 import org.jetbrains.annotations.NotNull;
 
 import java.util.List;
@@ -59,11 +59,11 @@ public class CCPushCourse extends DumbAwareAction {
         public void run(@NotNull ProgressIndicator indicator) {
           for (Lesson lesson : course.getLessons()) {
             if (lesson.getId() > 0) {
-              CCStepicConnector.updateLesson(project, lesson, indicator);
+              CCStepicConnector.updateLesson(project, lesson);
             }
             else {
               final CourseInfo info = CourseInfo.fromCourse(course);
-              final int lessonId = CCStepicConnector.postLesson(project, lesson, indicator);
+              final int lessonId = CCStepicConnector.postLesson(project, lesson);
               if (lessonId != -1) {
                 final List<Integer> sections = info.getSections();
                 final Integer sectionId = sections.get(sections.size() - 1);

python/educational-core/src/com/jetbrains/edu/coursecreator/actions/CCPushLesson.java

@@ -12,11 +12,11 @@ import com.intellij.openapi.project.DumbAwareAction;
 import com.intellij.openapi.project.Project;
 import com.intellij.psi.PsiDirectory;
 import com.jetbrains.edu.coursecreator.CCUtils;
+import com.jetbrains.edu.coursecreator.stepik.CCStepicConnector;
 import com.jetbrains.edu.learning.StudyTaskManager;
 import com.jetbrains.edu.learning.courseFormat.Course;
-import com.jetbrains.edu.learning.courseFormat.Lesson;
-import com.jetbrains.edu.coursecreator.stepik.CCStepicConnector;
 import com.jetbrains.edu.learning.courseFormat.CourseInfo;
+import com.jetbrains.edu.learning.courseFormat.Lesson;
 import com.jetbrains.edu.learning.stepic.EduStepicNames;
 import org.jetbrains.annotations.NotNull;
 
@@ -77,11 +77,11 @@ public class CCPushLesson extends DumbAwareAction {
       public void run(@NotNull ProgressIndicator indicator) {
         indicator.setText("Uploading lesson to " + EduStepicNames.STEPIC_URL);
         if (lesson.getId() > 0) {
-          CCStepicConnector.updateLesson(project, lesson, indicator);
+          CCStepicConnector.updateLesson(project, lesson);
         }
         else {
           final CourseInfo info = CourseInfo.fromCourse(course);
-          final int lessonId = CCStepicConnector.postLesson(project, lesson, indicator);
+          final int lessonId = CCStepicConnector.postLesson(project, lesson);
           final List<Integer> sections = info.getSections();
           final Integer sectionId = sections.get(sections.size()-1);
           CCStepicConnector.postUnit(lessonId, lesson.getIndex(), sectionId);

python/educational-core/src/com/jetbrains/edu/coursecreator/actions/CCShowPreview.java

@@ -31,10 +31,10 @@ import com.intellij.openapi.project.Project;
 import com.intellij.openapi.ui.FrameWrapper;
 import com.intellij.openapi.ui.Messages;
 import com.intellij.openapi.util.Disposer;
-import com.intellij.openapi.util.Pair;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.PsiDirectory;
 import com.intellij.psi.PsiFile;
+import com.intellij.testFramework.LightVirtualFile;
 import com.intellij.ui.JBColor;
 import com.jetbrains.edu.coursecreator.CCUtils;
 import com.jetbrains.edu.learning.StudyTaskManager;
@@ -109,33 +109,24 @@ public class CCShowPreview extends DumbAwareAction {
       return;
     }
 
-
     if (taskFile.getActivePlaceholders().isEmpty()) {
       Messages.showInfoMessage("Preview is available for task files with answer placeholders only", "No Preview for This File");
       return;
     }
 
-    VirtualFile generatedFilesFolder = CCUtils.getGeneratedFilesFolder(project, module);
-
-    if (generatedFilesFolder == null) {
-      return;
-    }
     final Task task = taskFile.getTask();
-    ApplicationManager.getApplication().runWriteAction(new Runnable() {
-      @Override
-      public void run() {
-        Pair<VirtualFile, TaskFile> pair =
-          EduUtils.createStudentFile(this, project, virtualFile, generatedFilesFolder, null,
-                                     task instanceof TaskWithSubtasks ? ((TaskWithSubtasks)task).getActiveSubtaskIndex() : 0);
-        if (pair != null) {
-          showPreviewDialog(project, pair.getFirst(), pair.getSecond());
-        }
+    ApplicationManager.getApplication().runWriteAction(() -> {
+      TaskFile studentTaskFile = EduUtils.createStudentFile(project, virtualFile, task.copy(),
+                                   task instanceof TaskWithSubtasks ? ((TaskWithSubtasks)task).getActiveSubtaskIndex() : 0);
+      if (studentTaskFile != null) {
+        showPreviewDialog(project, studentTaskFile);
       }
     });
   }
 
-  private static void showPreviewDialog(@NotNull Project project, @NotNull VirtualFile userFile, @NotNull TaskFile taskFile) {
+  private static void showPreviewDialog(@NotNull Project project, @NotNull TaskFile taskFile) {
     final FrameWrapper showPreviewFrame = new FrameWrapper(project);
+    final LightVirtualFile userFile = new LightVirtualFile(taskFile.name, taskFile.text);
     showPreviewFrame.setTitle(userFile.getName());
     LabeledEditor labeledEditor = new LabeledEditor(null);
     final EditorFactory factory = EditorFactory.getInstance();

python/educational-core/src/com/jetbrains/edu/coursecreator/intellij/EduCCModuleBuilder.java

@@ -28,7 +28,6 @@ import com.jetbrains.edu.coursecreator.ui.CCNewProjectPanel;
 import com.jetbrains.edu.learning.EduPluginConfigurator;
 import com.jetbrains.edu.learning.StudyTaskManager;
 import com.jetbrains.edu.learning.courseFormat.Course;
-import com.jetbrains.edu.learning.courseGeneration.StudyProjectGenerator;
 import com.jetbrains.edu.learning.intellij.generation.EduCourseModuleBuilder;
 import org.jdom.JDOMException;
 import org.jetbrains.annotations.NotNull;
@@ -36,7 +35,6 @@ import org.jetbrains.annotations.Nullable;
 
 import javax.swing.*;
 import javax.swing.text.JTextComponent;
-import java.io.File;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.function.Function;
@@ -93,8 +91,6 @@ class EduCCModuleBuilder extends EduCourseModuleBuilder {
     Language language = wrapper.getLanguage();
     course.setLanguage(language.getID());
     course.setCourseMode(CCUtils.COURSE_MODE);
-    File courseDir = new File(StudyProjectGenerator.OUR_COURSES_DIR, myPanel.getName() + "-" + project.getName());
-    course.setCourseDirectory(courseDir.getPath());
     StudyTaskManager.getInstance(project).setCourse(course);
     EduPluginConfigurator configurator = EduPluginConfigurator.INSTANCE.forLanguage(language);
     String languageName = language.getDisplayName();

python/educational-core/src/com/jetbrains/edu/coursecreator/settings/CCOptions.java

@@ -17,15 +17,13 @@ public class CCOptions implements StudyOptionsProvider {
   public JComponent createComponent() {
     if (CCSettings.getInstance().useHtmlAsDefaultTaskFormat()) {
       myHtmlRadioButton.setSelected(true);
-      IdeFocusManager.getGlobalInstance().doWhenFocusSettlesDown(() -> {
-        IdeFocusManager.getGlobalInstance().requestFocus(myHtmlRadioButton, true);
-      });
+      IdeFocusManager.getGlobalInstance().doWhenFocusSettlesDown(
+        () -> IdeFocusManager.getGlobalInstance().requestFocus(myHtmlRadioButton, true));
     }
     else {
       myMarkdownRadioButton.setSelected(true);
-      IdeFocusManager.getGlobalInstance().doWhenFocusSettlesDown(() -> {
-        IdeFocusManager.getGlobalInstance().requestFocus(myMarkdownRadioButton, true);
-      });
+      IdeFocusManager.getGlobalInstance().doWhenFocusSettlesDown(
+        () -> IdeFocusManager.getGlobalInstance().requestFocus(myMarkdownRadioButton, true));
     }
     return myPanel;
   }

python/educational-core/src/com/jetbrains/edu/coursecreator/stepik/CCStepicConnector.java

@@ -8,13 +8,9 @@ import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.progress.ProgressIndicator;
 import com.intellij.openapi.progress.ProgressManager;
 import com.intellij.openapi.project.Project;
-import com.intellij.openapi.util.io.FileUtil;
-import com.intellij.openapi.vfs.VfsUtil;
-import com.intellij.openapi.vfs.VirtualFile;
-import com.intellij.openapi.vfs.VirtualFileFilter;
+import com.jetbrains.edu.coursecreator.CCUtils;
 import com.jetbrains.edu.learning.StudySerializationUtils;
 import com.jetbrains.edu.learning.core.EduNames;
-import com.jetbrains.edu.learning.core.EduUtils;
 import com.jetbrains.edu.learning.courseFormat.AnswerPlaceholder;
 import com.jetbrains.edu.learning.courseFormat.Course;
 import com.jetbrains.edu.learning.courseFormat.CourseInfo;
@@ -24,7 +20,6 @@ import com.jetbrains.edu.learning.stepic.EduStepicAuthorizedClient;
 import com.jetbrains.edu.learning.stepic.EduStepicNames;
 import com.jetbrains.edu.learning.stepic.StepicUser;
 import com.jetbrains.edu.learning.stepic.StepicWrappers;
-import org.apache.commons.codec.binary.Base64;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpStatus;
 import org.apache.http.StatusLine;
@@ -65,13 +60,16 @@ public class CCStepicConnector {
     ProgressManager.getInstance().run(new com.intellij.openapi.progress.Task.Modal(project, "Uploading Course", true) {
       @Override
       public void run(@NotNull final ProgressIndicator indicator) {
-        postCourse(project, course, indicator);
+        postCourse(project, course);
       }
     });
   }
 
-  private static void postCourse(final Project project, @NotNull Course course, @NotNull final ProgressIndicator indicator) {
-    indicator.setText("Uploading course to " + EduStepicNames.STEPIC_URL);
+  private static void postCourse(final Project project, @NotNull Course course) {
+    final ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
+    if (indicator != null) {
+      indicator.setText("Uploading course to " + EduStepicNames.STEPIC_URL);
+    }
     final HttpPost request = new HttpPost(EduStepicNames.STEPIC_API_URL + "/courses");
 
     final StepicUser currentUser = EduStepicAuthorizedClient.getCurrentUser();
@@ -110,55 +108,25 @@ public class CCStepicConnector {
       final int sectionId = postModule(postedCourse.getId(), 1, String.valueOf(postedCourse.getName()));
       int position = 1;
       for (Lesson lesson : course.getLessons()) {
-        indicator.checkCanceled();
-        final int lessonId = postLesson(project, lesson, indicator);
+        if (indicator != null) {
+          indicator.checkCanceled();
+        }
+        final int lessonId = postLesson(project, lesson);
         postUnit(lessonId, position, sectionId);
         position += 1;
       }
-      ApplicationManager.getApplication().runReadAction(() -> postAdditionalFiles(project, postedCourse.getId(), indicator));
+      ApplicationManager.getApplication().runReadAction(() -> postAdditionalFiles(course, project, postedCourse.getId()));
     }
     catch (IOException e) {
       LOG.error(e.getMessage());
     }
   }
 
-  private static void postAdditionalFiles(@NotNull final Project project, int id, ProgressIndicator indicator) {
-    final VirtualFile baseDir = project.getBaseDir();
-    final List<VirtualFile> files = VfsUtil.getChildren(baseDir, new VirtualFileFilter() {
-      @Override
-      public boolean accept(VirtualFile file) {
-        final String name = file.getName();
-        return !name.contains(EduNames.LESSON) && !name.equals(EduNames.COURSE_META_FILE) && !name.equals(EduNames.HINTS) &&
-               !"pyc".equals(file.getExtension()) && !file.isDirectory() && !name.equals(EduNames.TEST_HELPER) && !name.startsWith(".");
-      }
-    });
-
-    if (!files.isEmpty()) {
+  private static void postAdditionalFiles(Course course, @NotNull final Project project, int id) {
+    final Lesson lesson = CCUtils.createAdditionalLesson(course, project);
+    if (lesson != null) {
       final int sectionId = postModule(id, 2, EduNames.PYCHARM_ADDITIONAL);
-      final Lesson lesson = new Lesson();
-      lesson.setName(EduNames.PYCHARM_ADDITIONAL);
-      final Task task = new Task();
-      task.setLesson(lesson);
-      task.setName(EduNames.PYCHARM_ADDITIONAL);
-      task.setIndex(1);
-      for (VirtualFile file : files) {
-        try {
-          if (file != null) {
-            if (EduUtils.isImage(file.getName())) {
-              task.addTestsTexts(file.getName(), Base64.encodeBase64URLSafeString(FileUtil.loadBytes(file.getInputStream())));
-            }
-            else {
-              task.addTestsTexts(file.getName(), FileUtil.loadTextAndClose(file.getInputStream()));
-            }
-          }
-        }
-        catch (IOException e) {
-          LOG.error("Can't find file " + file.getPath());
-        }
-      }
-      lesson.addTask(task);
-      lesson.setIndex(1);
-      final int lessonId = postLesson(project, lesson, indicator);
+      final int lessonId = postLesson(project, lesson);
       postUnit(lessonId, 1, sectionId);
     }
   }
@@ -255,7 +223,7 @@ public class CCStepicConnector {
     return -1;
   }
 
-  public static int updateLesson(@NotNull final Project project, @NotNull final Lesson lesson, ProgressIndicator indicator) {
+  public static int updateLesson(@NotNull final Project project, @NotNull final Lesson lesson) {
     final HttpPut request = new HttpPut(EduStepicNames.STEPIC_API_URL + EduStepicNames.LESSONS + String.valueOf(lesson.getId()));
 
     String requestBody = new Gson().toJson(new StepicWrappers.LessonWrapper(lesson));
@@ -279,7 +247,10 @@ public class CCStepicConnector {
       }
 
       for (Task task : lesson.getTaskList()) {
-        indicator.checkCanceled();
+        final ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
+        if (indicator != null) {
+          indicator.checkCanceled();
+        }
         postTask(project, task, lesson.getId());
       }
       return lesson.getId();
@@ -290,7 +261,7 @@ public class CCStepicConnector {
     return -1;
   }
 
-  public static int postLesson(@NotNull final Project project, @NotNull final Lesson lesson, ProgressIndicator indicator) {
+  public static int postLesson(@NotNull final Project project, @NotNull final Lesson lesson) {
     final HttpPost request = new HttpPost(EduStepicNames.STEPIC_API_URL + "/lessons");
 
     String requestBody = new Gson().toJson(new StepicWrappers.LessonWrapper(lesson));
@@ -311,7 +282,10 @@ public class CCStepicConnector {
       final Lesson postedLesson = new Gson().fromJson(responseString, Course.class).getLessons().get(0);
       lesson.setId(postedLesson.getId());
       for (Task task : lesson.getTaskList()) {
-        indicator.checkCanceled();
+        final ProgressIndicator indicator = ProgressManager.getInstance().getProgressIndicator();
+        if (indicator != null) {
+          indicator.checkCanceled();
+        }
         postTask(project, task, postedLesson.getId());
       }
       return postedLesson.getId();

python/educational-core/src/com/jetbrains/edu/learning/EduPluginConfigurator.java

@@ -21,8 +21,6 @@ import com.jetbrains.edu.learning.courseFormat.tasks.TaskWithSubtasks;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.io.File;
-
 public interface EduPluginConfigurator {
   String EP_NAME = "Edu.pluginConfigurator";
   LanguageExtension<EduPluginConfigurator> INSTANCE = new LanguageExtension<>(EP_NAME);
@@ -64,7 +62,7 @@ public interface EduPluginConfigurator {
   /**
    * Used in educator plugin to filter files to be packed into course archive
    */
-  boolean excludeFromArchive(@NotNull File pathname);
+  boolean excludeFromArchive(@NotNull String name);
 
   /**
    * @return true for all the test files including tests for subtasks
@@ -128,4 +126,7 @@ public interface EduPluginConfigurator {
                                          @Nullable String moduleDir) {
   }
 
+  default String getBundledCoursePath() {
+    return null;
+  }
 }

python/educational-core/src/com/jetbrains/edu/learning/StudyProjectComponent.java

@@ -39,9 +39,9 @@ import com.jetbrains.edu.learning.core.EduNames;
 import com.jetbrains.edu.learning.core.EduUtils;
 import com.jetbrains.edu.learning.courseFormat.*;
 import com.jetbrains.edu.learning.courseFormat.tasks.Task;
+import com.jetbrains.edu.learning.courseGeneration.StudyGenerator;
 import com.jetbrains.edu.learning.editor.StudyEditorFactoryListener;
 import com.jetbrains.edu.learning.statistics.EduUsagesCollector;
-import com.jetbrains.edu.learning.courseFormat.CourseInfo;
 import com.jetbrains.edu.learning.stepic.EduStepicConnector;
 import com.jetbrains.edu.learning.ui.StudyToolWindow;
 import com.jetbrains.edu.learning.ui.StudyToolWindowFactory;
@@ -50,14 +50,12 @@ import org.jetbrains.annotations.NotNull;
 
 import javax.swing.*;
 import javax.swing.event.HyperlinkEvent;
-import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
 import static com.jetbrains.edu.learning.StudyUtils.execCancelable;
-import static com.jetbrains.edu.learning.courseGeneration.StudyProjectGenerator.flushCourse;
 
 
 public class StudyProjectComponent implements ProjectComponent {
@@ -151,15 +149,8 @@ public class StudyProjectComponent implements ProjectComponent {
     final CourseInfo info = CourseInfo.fromCourse(currentCourse);
     if (info == null) return;
 
-    final File resourceDirectory = new File(currentCourse.getCourseDirectory());
-    if (resourceDirectory.exists()) {
-      FileUtil.delete(resourceDirectory);
-    }
-
     final Course course = EduStepicConnector.getCourse(myProject, info);
-
     if (course == null) return;
-    flushCourse(course);
     course.initCourse(false);
 
     EduPluginConfigurator configurator = EduPluginConfigurator.INSTANCE.forLanguage(course.getLanguageById());
@@ -176,14 +167,14 @@ public class StudyProjectComponent implements ProjectComponent {
       Lesson studentLesson = currentCourse.getLesson(lesson.getId());
       final String lessonDirName = EduNames.LESSON + String.valueOf(lessonIndex);
 
-      final File lessonDir = new File(myProject.getBasePath(), lessonDirName);
-      if (!lessonDir.exists()){
-        final File fromLesson = new File(resourceDirectory, lessonDirName);
+      final VirtualFile baseDir = myProject.getBaseDir();
+      final VirtualFile lessonDir = baseDir.findChild(lessonDirName);
+      if (lessonDir == null) {
         try {
-          FileUtil.copyDir(fromLesson, lessonDir);
+          StudyGenerator.createLesson(lesson, baseDir);
         }
         catch (IOException e) {
-          LOG.warn("Failed to copy lesson " + fromLesson.getPath());
+          LOG.error("Failed to create lesson");
         }
         lesson.setIndex(lessonIndex);
         lesson.initLesson(currentCourse, false);
@@ -211,15 +202,14 @@ public class StudyProjectComponent implements ProjectComponent {
         task.setIndex(index);
 
         final String taskDirName = EduNames.TASK + String.valueOf(index);
-        final File toTask = new File(lessonDir, taskDirName);
-
-        final String taskPath = FileUtil.join(resourceDirectory.getPath(), lessonDirName, taskDirName);
-        final File taskDir = new File(taskPath);
-        if (!taskDir.exists()) return;
-        final File[] taskFiles = taskDir.listFiles();
-        if (taskFiles == null) continue;
-        for (File fromFile : taskFiles) {
-          copyFile(fromFile, new File(toTask, fromFile.getName()));
+        final VirtualFile taskDir = lessonDir.findChild(taskDirName);
+
+        if (taskDir != null) return;
+        try {
+          StudyGenerator.createTask(task, lessonDir);
+        }
+        catch (IOException e) {
+          LOG.error("Failed to create task");
         }
         tasks.add(task);
       }
@@ -232,17 +222,6 @@ public class StudyProjectComponent implements ProjectComponent {
     notification.notify(myProject);
   }
 
-  private static void copyFile(@NotNull final File from, @NotNull final File to) {
-    if (from.exists()) {
-      try {
-        FileUtil.copyFileOrDir(from, to);
-      }
-      catch (IOException e) {
-        LOG.warn("Failed to copy " + from.getName());
-      }
-    }
-  }
-
   private void addShortcut(@NotNull final String actionIdString, @NotNull final String[] shortcuts) {
     KeymapManagerEx keymapManager = KeymapManagerEx.getInstanceEx();
     for (Keymap keymap : keymapManager.getAllKeymaps()) {

python/educational-core/src/com/jetbrains/edu/learning/StudySerializationUtils.java

@@ -528,9 +528,9 @@ public class StudySerializationUtils {
             stepOptionsJson = convertToSecondVersion(stepOptionsJson);
           case 2:
             stepOptionsJson = convertToThirdVersion(stepOptionsJson);
-            // uncomment for future versions
-            //case 3:
-            //  stepOptionsJson = convertToFourthVersion(stepOptionsJson);
+          // uncomment for future versions
+          //case 3:
+          //  stepOptionsJson = convertToFourthVersion(stepOptionsJson);
         }
         convertSubtaskInfosToMap(stepOptionsJson);
         StepicWrappers.StepOptions stepOptions =