1 /**
2 *
3 * Licensed to the Apache Software Foundation (ASF) under one
4 * or more contributor license agreements. See the NOTICE file
5 * distributed with this work for additional information
6 * regarding copyright ownership. The ASF licenses this file
7 * to you under the Apache License, Version 2.0 (the
8 * "License"); you may not use this file except in compliance
9 * with the License. You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19
20 package org.apache.hadoop.hbase.regionserver.wal;
21
22 import java.io.DataInput;
23 import java.io.DataOutput;
24 import java.io.IOException;
25 import java.util.List;
26 import java.util.UUID;
27 import java.util.concurrent.atomic.AtomicLong;
28 import java.util.regex.Pattern;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.hbase.classification.InterfaceAudience;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FSDataInputStream;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HRegionInfo;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.TableName;
40 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALTrailer;
41 import org.apache.hadoop.io.Writable;
42
43 import com.google.common.annotations.VisibleForTesting;
44
45
46 @InterfaceAudience.Private
47 // TODO: Rename interface to WAL
48 public interface HLog {
49 Log LOG = LogFactory.getLog(HLog.class);
50
51 /** File Extension used while splitting an HLog into regions (HBASE-2312) */
52 // TODO: this seems like an implementation detail that does not belong here.
53 String SPLITTING_EXT = "-splitting";
54 boolean SPLIT_SKIP_ERRORS_DEFAULT = false;
55 /** The hbase:meta region's HLog filename extension */
56 String META_HLOG_FILE_EXTN = ".meta";
57
58 /**
59 * Configuration name of HLog Trailer's warning size. If a waltrailer's size is greater than the
60 * configured size, a warning is logged. This is used with Protobuf reader/writer.
61 */
62 // TODO: Implementation detail. Why in here?
63 String WAL_TRAILER_WARN_SIZE = "hbase.regionserver.waltrailer.warn.size";
64 int DEFAULT_WAL_TRAILER_WARN_SIZE = 1024 * 1024; // 1MB
65
66 // TODO: Implemenation detail. Why in here?
67 Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");
68 String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";
69
70 interface Reader {
71
72 /**
73 * @param fs File system.
74 * @param path Path.
75 * @param c Configuration.
76 * @param s Input stream that may have been pre-opened by the caller; may be null.
77 */
78 void init(FileSystem fs, Path path, Configuration c, FSDataInputStream s) throws IOException;
79
80 void close() throws IOException;
81
82 Entry next() throws IOException;
83
84 Entry next(Entry reuse) throws IOException;
85
86 void seek(long pos) throws IOException;
87
88 long getPosition() throws IOException;
89 void reset() throws IOException;
90
91 /**
92 * @return the WALTrailer of the current HLog. It may be null in case of legacy or corrupt WAL
93 * files.
94 */
95 // TODO: What we need a trailer on WAL for?
96 WALTrailer getWALTrailer();
97 }
98
99 interface Writer {
100 void init(FileSystem fs, Path path, Configuration c, boolean overwritable) throws IOException;
101
102 void close() throws IOException;
103
104 void sync() throws IOException;
105
106 void append(Entry entry) throws IOException;
107
108 long getLength() throws IOException;
109
110 /**
111 * Sets HLog's WALTrailer. This trailer is appended at the end of WAL on closing.
112 * @param walTrailer trailer to append to WAL.
113 */
114 void setWALTrailer(WALTrailer walTrailer);
115 }
116
117 /**
118 * Utility class that lets us keep track of the edit with it's key.
119 * Only used when splitting logs.
120 */
121 // TODO: Remove this Writable.
122 class Entry implements Writable {
123 private WALEdit edit;
124 private HLogKey key;
125
126 public Entry() {
127 edit = new WALEdit();
128 key = new HLogKey();
129 }
130
131 /**
132 * Constructor for both params
133 *
134 * @param edit log's edit
135 * @param key log's key
136 */
137 public Entry(HLogKey key, WALEdit edit) {
138 super();
139 this.key = key;
140 this.edit = edit;
141 }
142
143 /**
144 * Gets the edit
145 *
146 * @return edit
147 */
148 public WALEdit getEdit() {
149 return edit;
150 }
151
152 /**
153 * Gets the key
154 *
155 * @return key
156 */
157 public HLogKey getKey() {
158 return key;
159 }
160
161 /**
162 * Set compression context for this entry.
163 *
164 * @param compressionContext
165 * Compression context
166 */
167 public void setCompressionContext(CompressionContext compressionContext) {
168 edit.setCompressionContext(compressionContext);
169 key.setCompressionContext(compressionContext);
170 }
171
172 @Override
173 public String toString() {
174 return this.key + "=" + this.edit;
175 }
176
177 @Override
178 @SuppressWarnings("deprecation")
179 public void write(DataOutput dataOutput) throws IOException {
180 this.key.write(dataOutput);
181 this.edit.write(dataOutput);
182 }
183
184 @Override
185 public void readFields(DataInput dataInput) throws IOException {
186 this.key.readFields(dataInput);
187 this.edit.readFields(dataInput);
188 }
189 }
190
191 /**
192 * registers WALActionsListener
193 *
194 * @param listener
195 */
196 void registerWALActionsListener(final WALActionsListener listener);
197
198 /**
199 * unregisters WALActionsListener
200 *
201 * @param listener
202 */
203 boolean unregisterWALActionsListener(final WALActionsListener listener);
204
205 /**
206 * @return Current state of the monotonically increasing file id.
207 */
208 // TODO: Remove. Implementation detail.
209 long getFilenum();
210
211 /**
212 * @return the number of HLog files
213 */
214 int getNumLogFiles();
215
216 /**
217 * @return the size of HLog files
218 */
219 long getLogFileSize();
220
221 // TODO: Log rolling should not be in this interface.
222 /**
223 * Roll the log writer. That is, start writing log messages to a new file.
224 *
225 * <p>
226 * The implementation is synchronized in order to make sure there's one rollWriter
227 * running at any given time.
228 *
229 * @return If lots of logs, flush the returned regions so next time through we
230 * can clean logs. Returns null if nothing to flush. Names are actual
231 * region names as returned by {@link HRegionInfo#getEncodedName()}
232 * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException
233 * @throws IOException
234 */
235 byte[][] rollWriter() throws FailedLogCloseException, IOException;
236
237 /**
238 * Roll the log writer. That is, start writing log messages to a new file.
239 *
240 * <p>
241 * The implementation is synchronized in order to make sure there's one rollWriter
242 * running at any given time.
243 *
244 * @param force
245 * If true, force creation of a new writer even if no entries have
246 * been written to the current writer
247 * @return If lots of logs, flush the returned regions so next time through we
248 * can clean logs. Returns null if nothing to flush. Names are actual
249 * region names as returned by {@link HRegionInfo#getEncodedName()}
250 * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException
251 * @throws IOException
252 */
253 byte[][] rollWriter(boolean force) throws FailedLogCloseException,
254 IOException;
255
256 /**
257 * Shut down the log.
258 *
259 * @throws IOException
260 */
261 void close() throws IOException;
262
263 /**
264 * Shut down the log and delete the log directory
265 *
266 * @throws IOException
267 */
268 void closeAndDelete() throws IOException;
269
270 /**
271 * Same as appendNoSync(HRegionInfo, TableName, WALEdit, List, long, HTableDescriptor),
272 * except it causes a sync on the log
273 * @param sequenceId of the region.
274 */
275 @VisibleForTesting
276 public void append(HRegionInfo info, TableName tableName, WALEdit edits,
277 final long now, HTableDescriptor htd, AtomicLong sequenceId) throws IOException;
278
279 /**
280 * For notification post append to the writer.
281 * @param entries
282 */
283 void postAppend(final List<Entry> entries);
284
285 /**
286 * For notification post writer sync.
287 */
288 void postSync();
289
290 /**
291 * Append a set of edits to the log. Log edits are keyed by (encoded) regionName, rowname, and
292 * log-sequence-id. The HLog is not flushed after this transaction is written to the log.
293 * @param info
294 * @param tableName
295 * @param edits
296 * @param clusterIds The clusters that have consumed the change (for replication)
297 * @param now
298 * @param htd
299 * @param sequenceId of the region
300 * @return txid of this transaction
301 * @throws IOException
302 */
303 long appendNoSync(HRegionInfo info, TableName tableName, WALEdit edits,
304 List<UUID> clusterIds, final long now, HTableDescriptor htd, AtomicLong sequenceId,
305 boolean isInMemstore, long nonceGroup, long nonce) throws IOException;
306
307 // TODO: Do we need all these versions of sync?
308 void hsync() throws IOException;
309
310 void hflush() throws IOException;
311
312 void sync() throws IOException;
313
314 void sync(long txid) throws IOException;
315
316 /**
317 * WAL keeps track of the sequence numbers that were not yet flushed from memstores
318 * in order to be able to do cleanup. This method tells WAL that some region is about
319 * to flush memstore.
320 *
321 * We stash the oldest seqNum for the region, and let the the next edit inserted in this
322 * region be recorded in {@link #append(HRegionInfo, TableName, WALEdit, long, HTableDescriptor,
323 * AtomicLong)} as new oldest seqnum.
324 * In case of flush being aborted, we put the stashed value back; in case of flush succeeding,
325 * the seqNum of that first edit after start becomes the valid oldest seqNum for this region.
326 *
327 * @return true if the flush can proceed, false in case wal is closing (ususally, when server is
328 * closing) and flush couldn't be started.
329 */
330 boolean startCacheFlush(final byte[] encodedRegionName);
331
332 /**
333 * Complete the cache flush.
334 * @param encodedRegionName Encoded region name.
335 */
336 void completeCacheFlush(final byte[] encodedRegionName);
337
338 /**
339 * Abort a cache flush. Call if the flush fails. Note that the only recovery
340 * for an aborted flush currently is a restart of the regionserver so the
341 * snapshot content dropped by the failure gets restored to the memstore.v
342 * @param encodedRegionName Encoded region name.
343 */
344 void abortCacheFlush(byte[] encodedRegionName);
345
346 /**
347 * @return Coprocessor host.
348 */
349 WALCoprocessorHost getCoprocessorHost();
350
351 /**
352 * Get LowReplication-Roller status
353 *
354 * @return lowReplicationRollEnabled
355 */
356 // TODO: This is implementation detail?
357 boolean isLowReplicationRollEnabled();
358
359 /** Gets the earliest sequence number in the memstore for this particular region.
360 * This can serve as best-effort "recent" WAL number for this region.
361 * @param encodedRegionName The region to get the number for.
362 * @return The number if present, HConstants.NO_SEQNUM if absent.
363 */
364 long getEarliestMemstoreSeqNum(byte[] encodedRegionName);
365 }