Wiki source code of Porter

Last modified by Thomas Mortagne on 2018/02/05 18:03
Show last authors
1 {{groovy}}
2 import com.xpn.xwiki.doc.XWikiAttachment;
3 import com.xpn.xwiki.doc.XWikiDocument;
4 import com.xpn.xwiki.doc.DeletedAttachment;
5 import org.xwiki.store.legacy.doc.internal.ListAttachmentArchive;
6 import com.xpn.xwiki.store.XWikiAttachmentStoreInterface;
7 import org.xwiki.store.legacy.store.internal.FilesystemAttachmentStore;
8 import com.xpn.xwiki.store.AttachmentVersioningStore;
9 import org.xwiki.store.legacy.store.internal.XWikiHibernateTransaction;
10 import com.xpn.xwiki.store.AttachmentRecycleBinStore;
11 import com.xpn.xwiki.web.Utils;
12 import com.xpn.xwiki.XWikiContext;
13 import com.xpn.xwiki.XWiki;
14 import com.xpn.xwiki.XWikiException;
15 import org.xwiki.store.filesystem.internal.FilesystemStoreTools;
16 import org.apache.commons.lang.exception.ExceptionUtils;
17 import org.hibernate.ObjectNotFoundException;
18
19 public void printIntro(final XWiki wiki)
20 {
21 println("= Filesystem attachment storage porting script =");
22
23 println("== Step 1: Switch to Filesystem attachments. ==");
24
25 boolean ret = isFilesystemStore(wiki.getAttachmentStore()) \
26 && isFilesystemStore(wiki.getAttachmentVersioningStore()) \
27 && isFilesystemStore(wiki.getAttachmentRecycleBinStore());
28
29 println(getStoreLine(isFilesystemStore(wiki.getAttachmentStore()),
30 wiki.getAttachmentStore()));
31 println(getStoreLine(isFilesystemStore(wiki.getAttachmentVersioningStore()),
32 wiki.getAttachmentVersioningStore()));
33 println(getStoreLine(isFilesystemStore(wiki.getAttachmentRecycleBinStore()),
34 wiki.getAttachmentRecycleBinStore()));
35
36 if (!ret) {
37 println("Please edit your xwiki.cfg file by modifying "
38 + "the attachment store lines to read as follows:");
39 println("xwiki.store.attachment.hint = file");
40 println("xwiki.store.attachment.versioning.hint = file");
41 println("xwiki.store.attachment.recyclebin.hint = file");
42 println("Also make sure they are not commented out.");
43 }
44
45 println("== Step 2: Add this directory to your backup routine. ==");
46 println("This is your storage directory, "
47 + "when filesystem attachments are enabled you have to back this up "
48 + "as part of your data backup routine.\n");
49 println(Utils.getComponent(FilesystemStoreTools.class).storageDir.getAbsolutePath());
50
51 println("== Step 3: Copy attachments from database to filesystem. ==");
52 println("Now you are ready to copy the data over from your database to the filesystem. "
53 + "It is prudent to leave the attachments in the database since this is still experimental "
54 + "and in most situations the attachment data is not bothersome just sitting in the "
55 + "database. As such, this script contains no facility to delete entries from the database.");
56 println("If anything goes wrong in this function, it will fail with an error message, "
57 + "you should get the stack trace (probably in the log file) and keep it to "
58 + "confuse and humiliate the developers with. No harm should be done since this only loads "
59 + "from the database and only saves to the filesystem.");
60 }
61
62 if (checkVersion()) {
63 mainEntryPoint();
64 }
65
66 public boolean checkVersion()
67 {
68 String version = xwiki.version
69 String[] versionElements = version.split('[\\.-]')
70
71 int major = versionElements[0].toInteger()
72 if (major < 9)
73 return true
74
75 if (major == 9) {
76 int minor = versionElements[1].toInteger()
77 if (minor < 10) {
78 return true
79 }
80 }
81
82 println """
83 {{error}}
84 This script work only for versions older than 9.10-rc-1 (current is $xwiki.version). Since 9.10-rc-1 XWiki support mixed storage for attachments so there is no need to moved them anymore.
85 {{/error}}"""
86
87 return false
88 }
89
90 public void mainEntryPoint()
91 {
92 final XWikiContext xc = xcontext.getContext();
93 final XWiki wiki = xc.getWiki();
94
95 int startAt = 0;
96 int seconds = 20;
97 boolean dryRun = true;
98 boolean verbose = false;
99 boolean go = request.getMethod().equals("POST") && request.getParameter("doIt") != null;
100 boolean ajax = "plain".equals(request.getParameter("xpage"));
101 int endedAt = 0;
102
103 if (!ajax) {
104 this.printIntro(wiki);
105 }
106
107 if (go) {
108 startAt = Integer.parseInt(request.getParameter("startAt"));
109 seconds = Integer.parseInt(request.getParameter("seconds"));
110 dryRun = "on".equals(request.getParameter("dryRun"));
111 verbose = "on".equals(request.getParameter("verbose"));
112 endedAt = this.doIt(startAt, seconds, dryRun, verbose, ajax);
113 if (ajax) {
114 if (endedAt == -1) {
115 this.printFooter();
116 } else {
117 println("\n{{html clean=false}}<meta name='endedAt' content='"
118 + endedAt + "' />{{/html}}");
119 }
120 return;
121 }
122 }
123
124 this.printJs();
125
126 if (!go || endedAt > startAt) {
127 println("\n{{html clean=false}}");
128 println("<div id='javascriptSpace'></div>");
129 println("<form action='?time=" + System.nanoTime()
130 + "#doItForm' id='doItForm' method='POST'>");
131 println("<input type='hidden' name='seconds' value='" + seconds + "' />");
132 println("<input type='hidden' id='startAt' name='startAt' value='" + endedAt + "' />");
133 println("<dl>");
134 println("<dt><label for='dryRun'>Dry Run, don't save any files.</label></dt>");
135 println("<dd><input type='checkbox' name='dryRun' checked='"
136 + ((dryRun) ? "checked" : "") + "' /></dd>");
137
138 println("<dt><label for='verbose'>Verbose, show every attachment.</label></dt>");
139 println("<dd><input type='checkbox' name='verbose' checked='"
140 + ((verbose) ? "checked" : "") + "' /></dd>");
141 println("</dl>");
142 println("<input type='hidden' name='doIt' value='yes' />")
143 println("<input type='submit' name='start' value='"
144 + ((endedAt > startAt) ? "Continue" : "Start") + "' />");
145 println("</form>");
146 println("<style>#xwikidata { display:none }</style>")
147 println("{{/html}}\n");
148 return;
149 }
150
151 this.printFooter();
152 }
153
154 public void printFooter()
155 {
156 println("== Step 4: make sure everything is working. ==");
157 println("Check to make sure your attachments are still there, if an attachment is broken, "
158 + "it will appear to be there but on opening it will tell you the attachment does not exist. "
159 + "If something goes terribly wrong with filesystem attachments, you may have your old "
160 + "attachment system back simply by changing the lines in the xwiki.cfg file and restarting "
161 + "the container. HOWEVER: This will not preserve attachments which were uploaded **after** "
162 + "switching to filesystem attachments so as users edit the system you will become "
163 + "locked in to filesystem attachments unless a script is written to do the "
164 + "inverse of this one.");
165 println("Despite being experimental, the filesystem attachment storage is quite stable and the "
166 + "risk of actually losing something so that it is unrecoverable is very remote.");
167 println("NOTE: This must be run seperately in each subwiki.");
168 println("\n\nEnjoy and give feedback.");
169
170 }
171
172 public static boolean isFilesystemStore(final Object store)
173 {
174 return store.getClass().getName().contains("Filesystem");
175 }
176
177 public static String getStoreLine(final boolean isFilesystem, final Object store)
178 {
179 return "(%style='color:" + (isFilesystem ? "#080" : "#d00") + "'%)(((* " + store + ")))";
180 }
181
182 public void printJs()
183 {
184 println("""\n{{html}}<script>
185 var isAtBottom = function() {
186 var vp = document.viewport;
187 return vp.getScrollOffsets().top + vp.getHeight() + 30 > window.body.getHeight();
188 };
189
190 Event.observe(document, "dom:loaded", function() {
191 var doItForm = document.getElementById("doItForm");
192 var workspace = document.getElementById("javascriptSpace");
193 var params = {
194 xpage: "plain",
195 seconds: doItForm.seconds.value,
196 startAt: doItForm.startAt.value,
197 doIt: "yes"
198 };
199 var shouldContinue;
200
201 var doBlock = function(startAt) {
202 params.startAt = startAt;
203 new Ajax.Request(window.location.href, {
204 method: "POST",
205 parameters: params,
206 onSuccess: function(transport) {
207 var div = document.createElement("div");
208 div.innerHTML = transport.responseText;
209
210 var wasAtBottom = isAtBottom();
211 workspace.innerHTML += transport.responseText;
212 if (wasAtBottom || startAt == 0) {
213 doItForm.scrollTo();
214 }
215
216 var metas = div.getElementsByTagName("meta");
217 if (metas.length > 0 && shouldContinue) {
218 doBlock(metas[0].content);
219 return;
220 } else if (metas.length > 0) {
221 doItForm.startAt.value = metas[0].content;
222 doItForm.start.value = "Continue";
223 } else {
224 doItForm.parentNode.removeChild(doItForm);
225 }
226 reachedEnd();
227 }
228 });
229 };
230
231 var reachedEnd = function() {
232 Event.observe(doItForm, "submit", start);
233 Event.stopObserving(doItForm, "submit", stop);
234 };
235
236 var stop = function(ev) {
237 ev.stop();
238 shouldContinue = false;
239 reachedEnd();
240 };
241
242 var start = function(ev) {
243 ev.stop();
244 params.dryRun = (doItForm.dryRun.checked) ? "on" : undefined;
245 params.verbose = (doItForm.verbose.checked) ? "on" : undefined;
246 workspace.innerHTML = "";
247 shouldContinue = true;
248 doBlock(doItForm.startAt.value);
249 doItForm.start.value = "Abort";
250 Event.stopObserving(doItForm, "submit", start);
251 Event.observe(doItForm, "submit", stop);
252 };
253
254 Event.observe(doItForm, "submit", start);
255 });
256 </script>{{/html}}""");
257 }
258
259 /**
260 * Port all attachments from the original form to filesystem attachment store.
261 * Do one batch at a time so the user can see the progress and it won't time out.
262 *
263 * @param startAt the index of the first job to do, 0 to start at the beginning.
264 * @param seconds the number of seconds to run for before returning.
265 * @param dryRun don't save any files if this is true.
266 * @return the number of the first job which was not done. This can be startAt in the next call.
267 * -1 if the operation is complete.
268 */
269 public int doIt(int startAt, int seconds, boolean dryRun, boolean verbose, boolean ajax)
270 {
271 final XWikiContext xc = this.xcontext.getContext();
272 final XWiki wiki = xc.getWiki();
273 final FilesystemAttachmentStore fileAttachStore =
274 Utils.getComponent(XWikiAttachmentStoreInterface.class, "file");
275 final XWikiAttachmentStoreInterface defAttachStore =
276 Utils.getComponent(XWikiAttachmentStoreInterface.class, "hibernate");
277 final AttachmentVersioningStore defAttachmentVersioningStore =
278 Utils.getComponent(AttachmentVersioningStore.class, "hibernate");
279
280 int count = 0;
281 long stopTime = System.currentTimeMillis() + (1000 * seconds);
282
283 println("(%style='color:#00d'%)(((");
284 if (verbose && startAt == 0) {
285 println("\n== Attachments ==\n");
286 }
287 for (String docName : xwiki.searchDocuments("where 1=1")) {
288 if (count < startAt) {
289 count++;
290 continue;
291 } else if (stopTime < System.currentTimeMillis()) {
292 println(")))");
293 return count;
294 } else {
295 count++;
296 }
297 final XWikiDocument doc = xwiki.getDocument(docName).getDocument();
298 try {
299 final XWikiHibernateTransaction transaction = new XWikiHibernateTransaction(xc);
300
301 if (verbose) {
302 println("* [[" + doc.getFullName() + "]]");
303 }
304 // This is the equivilant of doc.getAttachmentList() but there is an issue which
305 // causes some attachments to be lost.
306 // http://jira.xwiki.org/browse/XWIKI-7936
307 def hql = "select att from XWikiAttachment att WHERE att.docId=?";
308 for (XWikiAttachment attach : xwiki.search(hql, [doc.getId()])) {
309 attach.setDoc(doc);
310 if (verbose) {
311 println("** [[attach:" + doc.getFullName() + "@" + attach.getFilename() + "]]");
312 }
313 try {
314 fileAttachStore.loadAttachmentContent(attach, xc, false);
315 continue;
316 } catch (XWikiException noAttachmentInFilesystem) {
317 // no such attachment, let's save it.
318 }
319 try {
320 try {
321 defAttachStore.loadAttachmentContent(attach, xc, true);
322 } catch (XWikiException notInHibernate) {
323 final Throwable wrapped = notInHibernate.getException();
324 if (wrapped && wrapped instanceof ObjectNotFoundException) {
325 println("(%style='color:#c80'%)(((");
326 println("Unable to find this attachment [[attach:" + attach.getFilename()
327 + "@" + doc.getFullName() + "]] in the database, "
328 + "it may be the result of database corruption from a failed "
329 + "upload prior to the fixing of "
330 + "[[http://jira.xwiki.org/browse/XWIKI-5055]]");
331 println("xwikiattachment_content.XWA_ID = " + attach.getId());
332 println(")))");
333 }
334 }
335 defAttachmentVersioningStore.loadArchive(attach, xc, true);
336 // http://jira.xwiki.org/jira/browse/XWIKI-6199
337 attach.getAttachment_archive().getVersions();
338 // Setup the runnables to do the save but don't save just yet.
339 fileAttachStore.getAttachmentContentSaveRunnable(attach, false, xc)
340 .runIn(transaction);
341 } catch (Exception e) {
342 println("(%style='color:#d00'%)(((");
343 println("Error in [" + doc.getFullName() + ":" + attach.getFilename() + "]"
344 + e.getMessage());
345 println(ExceptionUtils.getStackTrace(e));
346 println(")))");
347 }
348 }
349 if (!dryRun) {
350 transaction.start();
351 }
352 } catch (Exception e) {
353 println("(%style='color:#d00'%)(((");
354 println("Failed to port attachments for document [" + doc.getFullName() + "]");
355 println(e.getMessage());
356 println(ExceptionUtils.getStackTrace(e));
357 println(")))");
358 }
359 }
360 println(")))");
361 if (startAt < count) {
362 int attachCount =
363 xwiki.search("select count(at) from XWikiDocument as doc, "
364 + "XWikiAttachment as at where at.docId = doc.id").get(0).intValue();
365 println("**[" + attachCount + "] attachments.**");
366 }
367
368
369 // And now for the deleted attachments.
370 final AttachmentRecycleBinStore fileBin =
371 Utils.getComponent(AttachmentRecycleBinStore.class, "file");
372 final AttachmentRecycleBinStore defaultBin =
373 Utils.getComponent(AttachmentRecycleBinStore.class, "hibernate");
374
375 if (verbose && startAt < count) {
376 println("\n== Deleted Attachments ==\n");
377 }
378 List<Object[]> entries =
379 xwiki.search("select da.id, da.docName, da.filename, da.date from DeletedAttachment as da");
380 if (verbose && entries.size() == 0) {
381 println("\nnone.");
382 }
383 int delAttachCount = 0;
384 for (Object[] o : entries) {
385 long delAttachId = o[0];
386 final String docName = o[1];
387 final String fileName = o[2];
388 final Date deleteDate = o[3];
389 delAttachCount++;
390 if (count < startAt) {
391 if (verbose && !ajax) {
392 println("* [[" + docName + "]] - " + fileName);
393 }
394 count++;
395 continue;
396 } else if (stopTime < System.currentTimeMillis()) {
397 return count;
398 }
399 count++;
400
401 try {
402 final DeletedAttachment delAttach =
403 defaultBin.getDeletedAttachment(delAttachId, xc, true);
404 // Can't use the transaction runnable here because (non-filesystem)
405 // deleted attachments hold all content in memory so they have to be
406 // handled one at a time.
407 final XWikiAttachment attach = delAttach.restoreAttachment(null, xc);
408 try {
409 // http://jira.xwiki.org/jira/browse/XWIKI-6199
410 attach.getAttachment_archive().getVersions();
411 } catch (NullPointerException e) {
412 attach.setAttachment_archive(new ListAttachmentArchive([attach]));
413 }
414 if (!dryRun) {
415 fileBin.saveToRecycleBin(attach,
416 delAttach.getDeleter(),
417 delAttach.getDate(),
418 xc,
419 false);
420 }
421
422 if (verbose) {
423 println("* [[" + docName + "]]");
424 println("** " + fileName + " - Deleted: " + deleteDate);
425 }
426 } catch (Exception e) {
427 println("(%style='color:#d00'%)(((");
428 println("* [[" + docName + "]] - " + fileName + " - Deleted: " + deleteDate);
429 println(e.getMessage());
430 println(")))");
431 println(ExceptionUtils.getStackTrace(e));
432 }
433 }
434 println("\n**[" + delAttachCount + "] deleted attachments.**");
435 return -1;
436 }
437 {{/groovy}}