001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.action.hadoop;
016    
017    import java.io.IOException;
018    import java.net.URISyntaxException;
019    import java.util.List;
020    
021    import org.apache.hadoop.conf.Configuration;
022    import org.apache.hadoop.fs.FileStatus;
023    import org.apache.hadoop.fs.FileSystem;
024    import org.apache.hadoop.fs.Path;
025    import org.apache.hadoop.fs.permission.FsPermission;
026    import org.apache.oozie.action.ActionExecutor;
027    import org.apache.oozie.action.ActionExecutorException;
028    import org.apache.oozie.client.WorkflowAction;
029    import org.apache.oozie.service.HadoopAccessorException;
030    import org.apache.oozie.service.HadoopAccessorService;
031    import org.apache.oozie.service.Services;
032    import org.apache.oozie.util.XmlUtils;
033    import org.jdom.Element;
034    
035    /**
036     * File system action executor. <p/> This executes the file system mkdir, move and delete commands
037     */
038    public class FsActionExecutor extends ActionExecutor {
039    
040        public FsActionExecutor() {
041            super("fs");
042        }
043    
044        Path getPath(Element element, String attribute) {
045            String str = element.getAttributeValue(attribute).trim();
046            return new Path(str);
047        }
048    
049        void validatePath(Path path, boolean withScheme) throws ActionExecutorException {
050            String scheme = path.toUri().getScheme();
051            if (withScheme) {
052                if (scheme == null) {
053                    throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS001",
054                                                      "Missing scheme in path [{0}]", path);
055                }
056                else {
057                    if (!scheme.equals("hdfs")) {
058                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS002",
059                                                          "Scheme [{0}] not support in path [{1}]", scheme, path);
060                    }
061                }
062            }
063            else {
064                if (scheme != null) {
065                    throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS003",
066                                                      "Scheme [{0}] not allowed in path [{1}]", scheme, path);
067                }
068            }
069        }
070    
071        @SuppressWarnings("unchecked")
072        void doOperations(Context context, Element element) throws ActionExecutorException {
073            try {
074                FileSystem fs = context.getAppFileSystem();
075                boolean recovery = fs.exists(getRecoveryPath(context));
076                if (!recovery) {
077                    fs.mkdirs(getRecoveryPath(context));
078                }
079                for (Element commandElement : (List<Element>) element.getChildren()) {
080                    String command = commandElement.getName();
081                    if (command.equals("mkdir")) {
082                        Path path = getPath(commandElement, "path");
083                        mkdir(context, path);
084                    }
085                    else {
086                        if (command.equals("delete")) {
087                            Path path = getPath(commandElement, "path");
088                            delete(context, path);
089                        }
090                        else {
091                            if (command.equals("move")) {
092                                Path source = getPath(commandElement, "source");
093                                Path target = getPath(commandElement, "target");
094                                move(context, source, target, recovery);
095                            }
096                            else {
097                                if (command.equals("chmod")) {
098                                    Path path = getPath(commandElement, "path");
099                                    String str = commandElement.getAttributeValue("dir-files");
100                                    boolean dirFiles = (str == null) || Boolean.parseBoolean(str);
101                                    String permissionsMask = commandElement.getAttributeValue("permissions").trim();
102                                    chmod(context, path, permissionsMask, dirFiles);
103                                }
104                            }
105                        }
106                    }
107                }
108            }
109            catch (Exception ex) {
110                throw convertException(ex);
111            }
112        }
113    
114        /**
115         * @param path
116         * @param context
117         * @return FileSystem
118         * @throws HadoopAccessorException
119         */
120        private FileSystem getFileSystemFor(Path path, Context context) throws HadoopAccessorException {
121            String user = context.getWorkflow().getUser();
122            String group = context.getWorkflow().getGroup();
123            return Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, path.toUri(),
124                    new Configuration());
125        }
126    
127        /**
128         * @param path
129         * @param user
130         * @param group
131         * @return FileSystem
132         * @throws HadoopAccessorException
133         */
134        private FileSystem getFileSystemFor(Path path, String user, String group) throws HadoopAccessorException {
135            return Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, path.toUri(),
136                    new Configuration());
137        }
138    
139        void mkdir(Context context, Path path) throws ActionExecutorException {
140            try {
141                validatePath(path, true);
142                FileSystem fs = getFileSystemFor(path, context);
143    
144                if (!fs.exists(path)) {
145                    if (!fs.mkdirs(path)) {
146                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS004",
147                                                          "mkdir, path [{0}] could not create directory", path);
148                    }
149                }
150            }
151            catch (Exception ex) {
152                throw convertException(ex);
153            }
154        }
155    
156        /**
157         * Delete path
158         *
159         * @param context
160         * @param path
161         * @throws ActionExecutorException
162         */
163        public void delete(Context context, Path path) throws ActionExecutorException {
164            try {
165                validatePath(path, true);
166                FileSystem fs = getFileSystemFor(path, context);
167    
168                if (fs.exists(path)) {
169                    if (!fs.delete(path, true)) {
170                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS005",
171                                                          "delete, path [{0}] could not delete path", path);
172                    }
173                }
174            }
175            catch (Exception ex) {
176                throw convertException(ex);
177            }
178        }
179    
180        /**
181         * Delete path
182         *
183         * @param user
184         * @param group
185         * @param path
186         * @throws ActionExecutorException
187         */
188        public void delete(String user, String group, Path path) throws ActionExecutorException {
189            try {
190                validatePath(path, true);
191                FileSystem fs = getFileSystemFor(path, user, group);
192    
193                if (fs.exists(path)) {
194                    if (!fs.delete(path, true)) {
195                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS005",
196                                "delete, path [{0}] could not delete path", path);
197                    }
198                }
199            }
200            catch (Exception ex) {
201                throw convertException(ex);
202            }
203        }
204    
205        /**
206         * Move source to target
207         *
208         * @param context
209         * @param source
210         * @param target
211         * @param recovery
212         * @throws ActionExecutorException
213         */
214        public void move(Context context, Path source, Path target, boolean recovery) throws ActionExecutorException {
215            try {
216                validatePath(source, true);
217                validatePath(target, false);
218                FileSystem fs = getFileSystemFor(source, context);
219    
220                if (!fs.exists(source) && !recovery) {
221                    throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS006",
222                                                      "move, source path [{0}] does not exist", source);
223                }
224    
225                Path path = new Path(source, target);
226                if (fs.exists(path) && !recovery) {
227                    throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS007",
228                                                      "move, target path [{0}] already exists", target);
229                }
230    
231                if (!fs.rename(source, target) && !recovery) {
232                    throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS008",
233                                                      "move, could not move [{0}] to [{1}]", source, target);
234                }
235            }
236            catch (Exception ex) {
237                throw convertException(ex);
238            }
239        }
240    
241        void chmod(Context context, Path path, String permissions, boolean dirFiles) throws ActionExecutorException {
242            try {
243                validatePath(path, true);
244                FileSystem fs = getFileSystemFor(path, context);
245    
246                if (!fs.exists(path)) {
247                    throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS009",
248                                                      "chmod, path [{0}] does not exist", path);
249                }
250    
251                FileStatus pathStatus = fs.getFileStatus(path);
252    
253                Path[] paths;
254                if (dirFiles && pathStatus.isDir()) {
255                    FileStatus[] filesStatus = fs.listStatus(path);
256                    paths = new Path[filesStatus.length];
257                    for (int i = 0; i < filesStatus.length; i++) {
258                        paths[i] = filesStatus[i].getPath();
259                    }
260                }
261                else {
262                    paths = new Path[]{path};
263                }
264    
265                FsPermission newFsPermission = createShortPermission(permissions, path);
266                fs.setPermission(path, newFsPermission);
267                for (Path p : paths) {
268                    fs.setPermission(p, newFsPermission);
269                }
270            }
271            catch (Exception ex) {
272                throw convertException(ex);
273            }
274        }
275    
276        FsPermission createShortPermission(String permissions, Path path) throws ActionExecutorException {
277            if (permissions.length() == 3) {
278                char user = permissions.charAt(0);
279                char group = permissions.charAt(1);
280                char other = permissions.charAt(2);
281                int useri = user - '0';
282                int groupi = group - '0';
283                int otheri = other - '0';
284                int mask = useri * 100 + groupi * 10 + otheri;
285                short omask = Short.parseShort(Integer.toString(mask), 8);
286                return new FsPermission(omask);
287            }
288            else {
289                if (permissions.length() == 10) {
290                    return FsPermission.valueOf(permissions);
291                }
292                else {
293                    throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS010",
294                                                      "chmod, path [{0}] invalid permissions mask [{1}]", path, permissions);
295                }
296            }
297        }
298    
299        @Override
300        public void check(Context context, WorkflowAction action) throws ActionExecutorException {
301        }
302    
303        @Override
304        public void kill(Context context, WorkflowAction action) throws ActionExecutorException {
305        }
306    
307        @Override
308        public void start(Context context, WorkflowAction action) throws ActionExecutorException {
309            try {
310                context.setStartData("-", "-", "-");
311                Element actionXml = XmlUtils.parseXml(action.getConf());
312                doOperations(context, actionXml);
313                context.setExecutionData("OK", null);
314            }
315            catch (Exception ex) {
316                throw convertException(ex);
317            }
318        }
319    
320        @Override
321        public void end(Context context, WorkflowAction action) throws ActionExecutorException {
322            String externalStatus = action.getExternalStatus();
323            WorkflowAction.Status status = externalStatus.equals("OK") ? WorkflowAction.Status.OK :
324                                           WorkflowAction.Status.ERROR;
325            context.setEndData(status, getActionSignal(status));
326            if (!context.getProtoActionConf().getBoolean("oozie.action.keep.action.dir", false)) {
327                try {
328                    FileSystem fs = context.getAppFileSystem();
329                    fs.delete(context.getActionDir(), true);
330                }
331                catch (Exception ex) {
332                    throw convertException(ex);
333                }
334            }
335        }
336    
337        @Override
338        public boolean isCompleted(String externalStatus) {
339            return true;
340        }
341    
342        /**
343         * @param context
344         * @return
345         * @throws HadoopAccessorException
346         * @throws IOException
347         * @throws URISyntaxException
348         */
349        public Path getRecoveryPath(Context context) throws HadoopAccessorException, IOException, URISyntaxException {
350            return new Path(context.getActionDir(), "fs-" + context.getRecoveryId());
351        }
352    
353    }